2024-12-02 05:16:22,031 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-02 05:16:22,043 main DEBUG Took 0.010930 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-02 05:16:22,044 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-02 05:16:22,044 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-02 05:16:22,045 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-02 05:16:22,046 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 05:16:22,055 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-02 05:16:22,069 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 05:16:22,070 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 05:16:22,070 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 05:16:22,071 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 05:16:22,071 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 05:16:22,072 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 05:16:22,073 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 05:16:22,073 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 05:16:22,074 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 05:16:22,074 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 05:16:22,075 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 05:16:22,075 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 05:16:22,075 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 05:16:22,076 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 05:16:22,076 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 05:16:22,077 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 05:16:22,077 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 05:16:22,077 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 05:16:22,078 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 05:16:22,078 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 05:16:22,078 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 05:16:22,079 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 05:16:22,079 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 05:16:22,079 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 05:16:22,079 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 05:16:22,080 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-02 05:16:22,081 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 05:16:22,082 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-02 05:16:22,084 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-02 05:16:22,084 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-02 05:16:22,085 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-02 05:16:22,086 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-02 05:16:22,093 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-02 05:16:22,096 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-02 05:16:22,097 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-02 05:16:22,098 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-02 05:16:22,098 main DEBUG createAppenders(={Console}) 2024-12-02 05:16:22,099 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-02 05:16:22,099 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-02 05:16:22,099 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-02 05:16:22,100 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-02 05:16:22,100 main DEBUG OutputStream closed 2024-12-02 05:16:22,100 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-02 05:16:22,100 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-02 05:16:22,101 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-02 05:16:22,162 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-02 05:16:22,164 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-02 05:16:22,165 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-02 05:16:22,166 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-02 05:16:22,166 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-02 05:16:22,167 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-02 05:16:22,167 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-02 05:16:22,167 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-02 05:16:22,167 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-02 05:16:22,168 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-02 05:16:22,168 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-02 05:16:22,168 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-02 05:16:22,169 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-02 05:16:22,169 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-02 05:16:22,169 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-02 05:16:22,169 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-02 05:16:22,169 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-02 05:16:22,170 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-02 05:16:22,172 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02 05:16:22,173 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-02 05:16:22,173 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-02 05:16:22,173 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-02T05:16:22,360 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda 2024-12-02 05:16:22,363 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-02 05:16:22,363 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02T05:16:22,370 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-02T05:16:22,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T05:16:22,410 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/cluster_01f97ee8-23a4-f5a0-8cda-7a5425401614, deleteOnExit=true 2024-12-02T05:16:22,411 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T05:16:22,432 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/test.cache.data in system properties and HBase conf 2024-12-02T05:16:22,432 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T05:16:22,433 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/hadoop.log.dir in system properties and HBase conf 2024-12-02T05:16:22,434 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T05:16:22,434 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T05:16:22,434 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T05:16:22,529 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-02T05:16:22,616 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T05:16:22,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T05:16:22,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T05:16:22,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T05:16:22,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T05:16:22,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T05:16:22,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T05:16:22,622 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T05:16:22,622 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T05:16:22,622 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T05:16:22,622 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/nfs.dump.dir in system properties and HBase conf 2024-12-02T05:16:22,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/java.io.tmpdir in system properties and HBase conf 2024-12-02T05:16:22,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T05:16:22,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T05:16:22,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T05:16:24,694 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-02T05:16:24,775 INFO [Time-limited test {}] log.Log(170): Logging initialized @3423ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-02T05:16:24,839 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T05:16:24,892 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T05:16:24,909 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T05:16:24,910 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T05:16:24,911 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T05:16:24,921 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T05:16:24,923 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/hadoop.log.dir/,AVAILABLE} 2024-12-02T05:16:24,924 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T05:16:25,075 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/java.io.tmpdir/jetty-localhost-34415-hadoop-hdfs-3_4_1-tests_jar-_-any-13077328337886732082/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T05:16:25,087 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:34415} 2024-12-02T05:16:25,087 INFO [Time-limited test {}] server.Server(415): Started @3736ms 2024-12-02T05:16:27,992 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T05:16:28,002 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T05:16:28,004 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T05:16:28,004 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T05:16:28,004 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T05:16:28,007 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/hadoop.log.dir/,AVAILABLE} 2024-12-02T05:16:28,008 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T05:16:28,134 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7acb84b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/java.io.tmpdir/jetty-localhost-40917-hadoop-hdfs-3_4_1-tests_jar-_-any-6003735745590239279/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T05:16:28,134 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:40917} 2024-12-02T05:16:28,135 INFO [Time-limited test {}] server.Server(415): Started @6784ms 2024-12-02T05:16:28,190 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T05:16:31,131 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/cluster_01f97ee8-23a4-f5a0-8cda-7a5425401614/data/data2/current/BP-16702269-172.17.0.2-1733116583194/current, will proceed with Du for space computation calculation, 2024-12-02T05:16:31,131 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/cluster_01f97ee8-23a4-f5a0-8cda-7a5425401614/data/data1/current/BP-16702269-172.17.0.2-1733116583194/current, will proceed with Du for space computation calculation, 2024-12-02T05:16:31,162 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T05:16:31,214 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6dc138466eccb1ee with lease ID 0x6e3e9aaa9042f85b: Processing first storage report for DS-5a5c0e22-a56e-4f32-ae1e-a90082acf4bf from datanode DatanodeRegistration(127.0.0.1:34519, datanodeUuid=3d349b4e-6cdf-4ef7-bca2-2a33ed5b96a5, infoPort=40065, infoSecurePort=0, ipcPort=38713, storageInfo=lv=-57;cid=testClusterID;nsid=1512845439;c=1733116583194) 2024-12-02T05:16:31,215 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6dc138466eccb1ee with lease ID 0x6e3e9aaa9042f85b: from storage DS-5a5c0e22-a56e-4f32-ae1e-a90082acf4bf node DatanodeRegistration(127.0.0.1:34519, datanodeUuid=3d349b4e-6cdf-4ef7-bca2-2a33ed5b96a5, infoPort=40065, infoSecurePort=0, ipcPort=38713, storageInfo=lv=-57;cid=testClusterID;nsid=1512845439;c=1733116583194), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T05:16:31,216 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6dc138466eccb1ee with lease ID 0x6e3e9aaa9042f85b: Processing first storage report for DS-c9637341-0144-4cbd-ad67-8f76339f918e from datanode DatanodeRegistration(127.0.0.1:34519, datanodeUuid=3d349b4e-6cdf-4ef7-bca2-2a33ed5b96a5, infoPort=40065, infoSecurePort=0, ipcPort=38713, storageInfo=lv=-57;cid=testClusterID;nsid=1512845439;c=1733116583194) 2024-12-02T05:16:31,216 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6dc138466eccb1ee with lease ID 0x6e3e9aaa9042f85b: from storage DS-c9637341-0144-4cbd-ad67-8f76339f918e node DatanodeRegistration(127.0.0.1:34519, datanodeUuid=3d349b4e-6cdf-4ef7-bca2-2a33ed5b96a5, infoPort=40065, infoSecurePort=0, ipcPort=38713, storageInfo=lv=-57;cid=testClusterID;nsid=1512845439;c=1733116583194), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T05:16:31,282 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda 2024-12-02T05:16:31,943 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/cluster_01f97ee8-23a4-f5a0-8cda-7a5425401614/zookeeper_0, clientPort=50310, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/cluster_01f97ee8-23a4-f5a0-8cda-7a5425401614/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/cluster_01f97ee8-23a4-f5a0-8cda-7a5425401614/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T05:16:31,951 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50310 2024-12-02T05:16:31,958 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T05:16:31,961 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T05:16:32,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741825_1001 (size=7) 2024-12-02T05:16:32,943 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c with version=8 2024-12-02T05:16:32,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/hbase-staging 2024-12-02T05:16:33,017 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-02T05:16:33,201 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f49461030c22:0 server-side Connection retries=45 2024-12-02T05:16:33,209 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T05:16:33,209 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T05:16:33,213 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T05:16:33,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T05:16:33,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T05:16:33,319 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T05:16:33,376 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-02T05:16:33,387 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-02T05:16:33,392 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T05:16:33,423 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4303 (auto-detected) 2024-12-02T05:16:33,424 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-02T05:16:33,443 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42731 2024-12-02T05:16:33,464 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42731 connecting to ZooKeeper ensemble=127.0.0.1:50310 2024-12-02T05:16:33,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:427310x0, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T05:16:33,746 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42731-0x10194787add0000 connected 2024-12-02T05:16:34,348 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T05:16:34,350 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T05:16:34,361 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T05:16:34,365 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c, hbase.cluster.distributed=false 2024-12-02T05:16:34,387 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T05:16:34,392 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42731 2024-12-02T05:16:34,392 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42731 2024-12-02T05:16:34,392 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42731 2024-12-02T05:16:34,393 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42731 2024-12-02T05:16:34,393 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42731 2024-12-02T05:16:34,496 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f49461030c22:0 server-side Connection retries=45 2024-12-02T05:16:34,498 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T05:16:34,498 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T05:16:34,498 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T05:16:34,499 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T05:16:34,499 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T05:16:34,501 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T05:16:34,504 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T05:16:34,505 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46495 2024-12-02T05:16:34,506 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46495 connecting to ZooKeeper ensemble=127.0.0.1:50310 2024-12-02T05:16:34,508 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T05:16:34,511 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T05:16:34,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:464950x0, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T05:16:34,693 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46495-0x10194787add0001 connected 2024-12-02T05:16:34,694 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T05:16:34,700 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T05:16:34,709 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T05:16:34,712 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T05:16:34,717 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T05:16:34,718 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46495 2024-12-02T05:16:34,718 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46495 2024-12-02T05:16:34,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46495 2024-12-02T05:16:34,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46495 2024-12-02T05:16:34,720 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46495 2024-12-02T05:16:34,734 DEBUG [M:0;f49461030c22:42731 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f49461030c22:42731 2024-12-02T05:16:34,734 INFO [master/f49461030c22:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f49461030c22,42731,1733116593060 2024-12-02T05:16:34,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T05:16:34,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T05:16:34,963 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f49461030c22,42731,1733116593060 2024-12-02T05:16:35,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T05:16:35,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T05:16:35,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T05:16:35,133 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T05:16:35,135 INFO [master/f49461030c22:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f49461030c22,42731,1733116593060 from backup master directory 2024-12-02T05:16:35,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f49461030c22,42731,1733116593060 2024-12-02T05:16:35,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T05:16:35,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T05:16:35,343 WARN [master/f49461030c22:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T05:16:35,343 INFO [master/f49461030c22:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f49461030c22,42731,1733116593060 2024-12-02T05:16:35,348 INFO [master/f49461030c22:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-02T05:16:35,351 INFO [master/f49461030c22:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-02T05:16:35,404 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/hbase.id] with ID: e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21 2024-12-02T05:16:35,404 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/.tmp/hbase.id 2024-12-02T05:16:35,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741826_1002 (size=42) 2024-12-02T05:16:35,816 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/.tmp/hbase.id]:[hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/hbase.id] 2024-12-02T05:16:35,865 INFO [master/f49461030c22:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T05:16:35,870 INFO [master/f49461030c22:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T05:16:35,891 INFO [master/f49461030c22:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-12-02T05:16:36,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T05:16:36,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T05:16:36,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741827_1003 (size=196) 2024-12-02T05:16:36,547 INFO [master/f49461030c22:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T05:16:36,549 INFO [master/f49461030c22:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T05:16:36,566 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:36,571 INFO [master/f49461030c22:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T05:16:36,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741828_1004 (size=1189) 2024-12-02T05:16:37,021 INFO [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store 2024-12-02T05:16:37,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741829_1005 (size=34) 2024-12-02T05:16:37,445 INFO [master/f49461030c22:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-02T05:16:37,451 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:16:37,453 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T05:16:37,453 INFO [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T05:16:37,453 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T05:16:37,454 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T05:16:37,454 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T05:16:37,454 INFO [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T05:16:37,455 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733116597452Disabling compacts and flushes for region at 1733116597452Disabling writes for close at 1733116597454 (+2 ms)Writing region close event to WAL at 1733116597454Closed at 1733116597454 2024-12-02T05:16:37,457 WARN [master/f49461030c22:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/.initializing 2024-12-02T05:16:37,458 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/WALs/f49461030c22,42731,1733116593060 2024-12-02T05:16:37,465 INFO [master/f49461030c22:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T05:16:37,479 INFO [master/f49461030c22:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f49461030c22%2C42731%2C1733116593060, suffix=, logDir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/WALs/f49461030c22,42731,1733116593060, archiveDir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/oldWALs, maxLogs=10 2024-12-02T05:16:37,499 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/WALs/f49461030c22,42731,1733116593060/f49461030c22%2C42731%2C1733116593060.1733116597483, exclude list is [], retry=0 2024-12-02T05:16:37,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34519,DS-5a5c0e22-a56e-4f32-ae1e-a90082acf4bf,DISK] 2024-12-02T05:16:37,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-02T05:16:37,550 INFO [master/f49461030c22:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/WALs/f49461030c22,42731,1733116593060/f49461030c22%2C42731%2C1733116593060.1733116597483 2024-12-02T05:16:37,551 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40065:40065)] 2024-12-02T05:16:37,551 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T05:16:37,552 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:16:37,555 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T05:16:37,556 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T05:16:37,594 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T05:16:37,618 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T05:16:37,622 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:37,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T05:16:37,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T05:16:37,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T05:16:37,629 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:37,630 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:16:37,630 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T05:16:37,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T05:16:37,633 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:37,634 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:16:37,635 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T05:16:37,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T05:16:37,638 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:37,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:16:37,639 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T05:16:37,643 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T05:16:37,644 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T05:16:37,649 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T05:16:37,650 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T05:16:37,653 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T05:16:37,657 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T05:16:37,662 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T05:16:37,663 INFO [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68742549, jitterRate=0.02434380352497101}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T05:16:37,669 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733116597568Initializing all the Stores at 1733116597571 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733116597571Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116597572 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116597572Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116597572Cleaning up temporary data from old regions at 1733116597650 (+78 ms)Region opened successfully at 1733116597669 (+19 ms) 2024-12-02T05:16:37,671 INFO [master/f49461030c22:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T05:16:37,703 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13bba31b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f49461030c22/172.17.0.2:0 2024-12-02T05:16:37,736 INFO [master/f49461030c22:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T05:16:37,747 INFO [master/f49461030c22:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T05:16:37,747 INFO [master/f49461030c22:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T05:16:37,749 INFO [master/f49461030c22:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T05:16:37,751 INFO [master/f49461030c22:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-02T05:16:37,755 INFO [master/f49461030c22:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-02T05:16:37,755 INFO [master/f49461030c22:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T05:16:37,779 INFO [master/f49461030c22:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T05:16:37,788 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T05:16:38,005 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T05:16:38,011 INFO [master/f49461030c22:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T05:16:38,014 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T05:16:38,174 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T05:16:38,181 INFO [master/f49461030c22:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T05:16:38,188 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T05:16:38,321 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T05:16:38,325 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T05:16:38,342 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T05:16:38,364 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T05:16:38,574 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T05:16:38,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T05:16:38,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T05:16:38,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T05:16:38,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T05:16:38,618 INFO [master/f49461030c22:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f49461030c22,42731,1733116593060, sessionid=0x10194787add0000, setting cluster-up flag (Was=false) 2024-12-02T05:16:38,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T05:16:38,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T05:16:38,881 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T05:16:38,884 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f49461030c22,42731,1733116593060 2024-12-02T05:16:39,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T05:16:39,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T05:16:39,154 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T05:16:39,157 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f49461030c22,42731,1733116593060 2024-12-02T05:16:39,165 INFO [master/f49461030c22:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T05:16:39,228 INFO [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(746): ClusterId : e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21 2024-12-02T05:16:39,231 DEBUG [RS:0;f49461030c22:46495 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T05:16:39,235 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T05:16:39,245 INFO [master/f49461030c22:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T05:16:39,252 INFO [master/f49461030c22:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T05:16:39,258 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f49461030c22,42731,1733116593060 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T05:16:39,459 DEBUG [RS:0;f49461030c22:46495 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T05:16:39,459 DEBUG [RS:0;f49461030c22:46495 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T05:16:39,460 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f49461030c22:0, corePoolSize=5, maxPoolSize=5 2024-12-02T05:16:39,460 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f49461030c22:0, corePoolSize=5, maxPoolSize=5 2024-12-02T05:16:39,461 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f49461030c22:0, corePoolSize=5, maxPoolSize=5 2024-12-02T05:16:39,461 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f49461030c22:0, corePoolSize=5, maxPoolSize=5 2024-12-02T05:16:39,462 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f49461030c22:0, corePoolSize=10, maxPoolSize=10 2024-12-02T05:16:39,462 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f49461030c22:0, corePoolSize=1, maxPoolSize=1 2024-12-02T05:16:39,463 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f49461030c22:0, corePoolSize=2, maxPoolSize=2 2024-12-02T05:16:39,463 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f49461030c22:0, corePoolSize=1, maxPoolSize=1 2024-12-02T05:16:39,464 INFO [master/f49461030c22:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733116629464 2024-12-02T05:16:39,466 INFO [master/f49461030c22:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T05:16:39,467 INFO [master/f49461030c22:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T05:16:39,468 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T05:16:39,469 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T05:16:39,471 INFO [master/f49461030c22:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T05:16:39,471 INFO [master/f49461030c22:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T05:16:39,472 INFO [master/f49461030c22:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T05:16:39,472 INFO [master/f49461030c22:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T05:16:39,473 INFO [master/f49461030c22:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:39,475 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:39,476 INFO [master/f49461030c22:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T05:16:39,476 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T05:16:39,477 INFO [master/f49461030c22:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T05:16:39,477 INFO [master/f49461030c22:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T05:16:39,479 INFO [master/f49461030c22:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T05:16:39,480 INFO [master/f49461030c22:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T05:16:39,482 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f49461030c22:0:becomeActiveMaster-HFileCleaner.large.0-1733116599481,5,FailOnTimeoutGroup] 2024-12-02T05:16:39,482 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f49461030c22:0:becomeActiveMaster-HFileCleaner.small.0-1733116599482,5,FailOnTimeoutGroup] 2024-12-02T05:16:39,482 INFO [master/f49461030c22:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:39,483 INFO [master/f49461030c22:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T05:16:39,484 INFO [master/f49461030c22:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:39,484 INFO [master/f49461030c22:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:39,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741831_1007 (size=1321) 2024-12-02T05:16:39,493 DEBUG [RS:0;f49461030c22:46495 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T05:16:39,493 DEBUG [RS:0;f49461030c22:46495 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c049e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f49461030c22/172.17.0.2:0 2024-12-02T05:16:39,508 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f49461030c22:46495 2024-12-02T05:16:39,512 INFO [RS:0;f49461030c22:46495 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T05:16:39,512 INFO [RS:0;f49461030c22:46495 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T05:16:39,512 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T05:16:39,515 INFO [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(2659): reportForDuty to master=f49461030c22,42731,1733116593060 with port=46495, startcode=1733116594459 2024-12-02T05:16:39,523 DEBUG [RS:0;f49461030c22:46495 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T05:16:39,552 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51465, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T05:16:39,557 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42731 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f49461030c22,46495,1733116594459 2024-12-02T05:16:39,559 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42731 {}] master.ServerManager(517): Registering regionserver=f49461030c22,46495,1733116594459 2024-12-02T05:16:39,572 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c 2024-12-02T05:16:39,572 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41031 2024-12-02T05:16:39,572 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T05:16:39,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T05:16:39,766 DEBUG [RS:0;f49461030c22:46495 {}] zookeeper.ZKUtil(111): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f49461030c22,46495,1733116594459 2024-12-02T05:16:39,766 WARN [RS:0;f49461030c22:46495 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T05:16:39,767 INFO [RS:0;f49461030c22:46495 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T05:16:39,768 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/WALs/f49461030c22,46495,1733116594459 2024-12-02T05:16:39,771 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f49461030c22,46495,1733116594459] 2024-12-02T05:16:39,795 INFO [RS:0;f49461030c22:46495 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T05:16:39,807 INFO [RS:0;f49461030c22:46495 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T05:16:39,812 INFO [RS:0;f49461030c22:46495 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T05:16:39,812 INFO [RS:0;f49461030c22:46495 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:39,814 INFO [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T05:16:39,819 INFO [RS:0;f49461030c22:46495 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T05:16:39,820 INFO [RS:0;f49461030c22:46495 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:39,821 DEBUG [RS:0;f49461030c22:46495 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f49461030c22:0, corePoolSize=1, maxPoolSize=1 2024-12-02T05:16:39,821 DEBUG [RS:0;f49461030c22:46495 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f49461030c22:0, corePoolSize=1, maxPoolSize=1 2024-12-02T05:16:39,821 DEBUG [RS:0;f49461030c22:46495 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f49461030c22:0, corePoolSize=1, maxPoolSize=1 2024-12-02T05:16:39,821 DEBUG [RS:0;f49461030c22:46495 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f49461030c22:0, corePoolSize=1, maxPoolSize=1 2024-12-02T05:16:39,821 DEBUG [RS:0;f49461030c22:46495 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f49461030c22:0, corePoolSize=1, maxPoolSize=1 2024-12-02T05:16:39,822 DEBUG [RS:0;f49461030c22:46495 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f49461030c22:0, corePoolSize=2, maxPoolSize=2 2024-12-02T05:16:39,822 DEBUG [RS:0;f49461030c22:46495 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0, corePoolSize=1, maxPoolSize=1 2024-12-02T05:16:39,822 DEBUG [RS:0;f49461030c22:46495 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f49461030c22:0, corePoolSize=1, maxPoolSize=1 2024-12-02T05:16:39,822 DEBUG [RS:0;f49461030c22:46495 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f49461030c22:0, corePoolSize=1, maxPoolSize=1 2024-12-02T05:16:39,822 DEBUG [RS:0;f49461030c22:46495 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f49461030c22:0, corePoolSize=1, maxPoolSize=1 2024-12-02T05:16:39,822 DEBUG [RS:0;f49461030c22:46495 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f49461030c22:0, corePoolSize=1, maxPoolSize=1 2024-12-02T05:16:39,822 DEBUG [RS:0;f49461030c22:46495 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f49461030c22:0, corePoolSize=1, maxPoolSize=1 2024-12-02T05:16:39,823 DEBUG [RS:0;f49461030c22:46495 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f49461030c22:0, corePoolSize=3, maxPoolSize=3 2024-12-02T05:16:39,823 DEBUG [RS:0;f49461030c22:46495 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0, corePoolSize=3, maxPoolSize=3 2024-12-02T05:16:39,824 INFO [RS:0;f49461030c22:46495 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:39,824 INFO [RS:0;f49461030c22:46495 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:39,824 INFO [RS:0;f49461030c22:46495 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:39,824 INFO [RS:0;f49461030c22:46495 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:39,824 INFO [RS:0;f49461030c22:46495 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:39,824 INFO [RS:0;f49461030c22:46495 {}] hbase.ChoreService(168): Chore ScheduledChore name=f49461030c22,46495,1733116594459-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T05:16:39,842 INFO [RS:0;f49461030c22:46495 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T05:16:39,843 INFO [RS:0;f49461030c22:46495 {}] hbase.ChoreService(168): Chore ScheduledChore name=f49461030c22,46495,1733116594459-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:39,843 INFO [RS:0;f49461030c22:46495 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:39,844 INFO [RS:0;f49461030c22:46495 {}] regionserver.Replication(171): f49461030c22,46495,1733116594459 started 2024-12-02T05:16:39,866 INFO [RS:0;f49461030c22:46495 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:39,867 INFO [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1482): Serving as f49461030c22,46495,1733116594459, RpcServer on f49461030c22/172.17.0.2:46495, sessionid=0x10194787add0001 2024-12-02T05:16:39,868 DEBUG [RS:0;f49461030c22:46495 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T05:16:39,868 DEBUG [RS:0;f49461030c22:46495 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f49461030c22,46495,1733116594459 2024-12-02T05:16:39,868 DEBUG [RS:0;f49461030c22:46495 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f49461030c22,46495,1733116594459' 2024-12-02T05:16:39,869 DEBUG [RS:0;f49461030c22:46495 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T05:16:39,870 DEBUG [RS:0;f49461030c22:46495 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T05:16:39,871 DEBUG [RS:0;f49461030c22:46495 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T05:16:39,871 DEBUG [RS:0;f49461030c22:46495 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T05:16:39,871 DEBUG [RS:0;f49461030c22:46495 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f49461030c22,46495,1733116594459 2024-12-02T05:16:39,871 DEBUG [RS:0;f49461030c22:46495 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f49461030c22,46495,1733116594459' 2024-12-02T05:16:39,871 DEBUG [RS:0;f49461030c22:46495 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T05:16:39,872 DEBUG [RS:0;f49461030c22:46495 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T05:16:39,873 DEBUG [RS:0;f49461030c22:46495 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T05:16:39,873 INFO [RS:0;f49461030c22:46495 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T05:16:39,873 INFO [RS:0;f49461030c22:46495 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T05:16:39,887 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T05:16:39,888 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c 2024-12-02T05:16:39,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741832_1008 (size=32) 2024-12-02T05:16:39,984 INFO [RS:0;f49461030c22:46495 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T05:16:39,988 INFO [RS:0;f49461030c22:46495 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f49461030c22%2C46495%2C1733116594459, suffix=, logDir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/WALs/f49461030c22,46495,1733116594459, archiveDir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/oldWALs, maxLogs=32 2024-12-02T05:16:40,001 DEBUG [RS:0;f49461030c22:46495 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/WALs/f49461030c22,46495,1733116594459/f49461030c22%2C46495%2C1733116594459.1733116599990, exclude list is [], retry=0 2024-12-02T05:16:40,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34519,DS-5a5c0e22-a56e-4f32-ae1e-a90082acf4bf,DISK] 2024-12-02T05:16:40,010 INFO [RS:0;f49461030c22:46495 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/WALs/f49461030c22,46495,1733116594459/f49461030c22%2C46495%2C1733116594459.1733116599990 2024-12-02T05:16:40,010 DEBUG [RS:0;f49461030c22:46495 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40065:40065)] 2024-12-02T05:16:40,301 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:16:40,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T05:16:40,309 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T05:16:40,309 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:40,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T05:16:40,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T05:16:40,312 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T05:16:40,313 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:40,313 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T05:16:40,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T05:16:40,316 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T05:16:40,316 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:40,317 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T05:16:40,317 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T05:16:40,320 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T05:16:40,320 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:40,321 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T05:16:40,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T05:16:40,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740 2024-12-02T05:16:40,324 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740 2024-12-02T05:16:40,326 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T05:16:40,326 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T05:16:40,327 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T05:16:40,329 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T05:16:40,333 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T05:16:40,334 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70431145, jitterRate=0.049505844712257385}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T05:16:40,336 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733116600301Initializing all the Stores at 1733116600304 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733116600304Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733116600305 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116600305Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733116600305Cleaning up temporary data from old regions at 1733116600326 (+21 ms)Region opened successfully at 1733116600336 (+10 ms) 2024-12-02T05:16:40,336 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T05:16:40,336 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T05:16:40,336 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T05:16:40,337 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T05:16:40,337 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T05:16:40,338 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T05:16:40,338 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733116600336Disabling compacts and flushes for region at 1733116600336Disabling writes for close at 1733116600337 (+1 ms)Writing region close event to WAL at 1733116600337Closed at 1733116600338 (+1 ms) 2024-12-02T05:16:40,341 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T05:16:40,341 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T05:16:40,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T05:16:40,356 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T05:16:40,359 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T05:16:40,513 DEBUG [f49461030c22:42731 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-02T05:16:40,528 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:16:40,535 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f49461030c22,46495,1733116594459, state=OPENING 2024-12-02T05:16:40,763 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T05:16:40,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T05:16:40,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T05:16:40,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T05:16:40,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T05:16:40,780 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T05:16:40,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:16:40,964 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T05:16:40,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34533, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T05:16:40,982 INFO [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T05:16:40,982 INFO [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T05:16:40,983 INFO [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T05:16:40,986 INFO [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f49461030c22%2C46495%2C1733116594459.meta, suffix=.meta, logDir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/WALs/f49461030c22,46495,1733116594459, archiveDir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/oldWALs, maxLogs=32 2024-12-02T05:16:41,001 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/WALs/f49461030c22,46495,1733116594459/f49461030c22%2C46495%2C1733116594459.meta.1733116600988.meta, exclude list is [], retry=0 2024-12-02T05:16:41,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34519,DS-5a5c0e22-a56e-4f32-ae1e-a90082acf4bf,DISK] 2024-12-02T05:16:41,009 INFO [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/WALs/f49461030c22,46495,1733116594459/f49461030c22%2C46495%2C1733116594459.meta.1733116600988.meta 2024-12-02T05:16:41,009 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40065:40065)] 2024-12-02T05:16:41,009 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T05:16:41,011 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T05:16:41,014 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T05:16:41,018 INFO [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T05:16:41,022 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T05:16:41,022 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:16:41,023 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T05:16:41,023 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T05:16:41,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T05:16:41,028 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T05:16:41,028 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:41,029 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T05:16:41,029 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T05:16:41,030 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T05:16:41,030 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:41,031 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T05:16:41,031 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T05:16:41,033 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T05:16:41,033 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:41,034 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T05:16:41,034 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T05:16:41,035 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T05:16:41,035 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:41,036 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T05:16:41,037 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T05:16:41,038 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740 2024-12-02T05:16:41,041 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740 2024-12-02T05:16:41,043 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T05:16:41,043 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T05:16:41,044 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T05:16:41,047 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T05:16:41,048 INFO [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72979477, jitterRate=0.08747895061969757}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T05:16:41,049 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T05:16:41,050 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733116601023Writing region info on filesystem at 1733116601024 (+1 ms)Initializing all the Stores at 1733116601025 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733116601026 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733116601026Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116601026Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733116601026Cleaning up temporary data from old regions at 1733116601044 (+18 ms)Running coprocessor post-open hooks at 1733116601049 (+5 ms)Region opened successfully at 1733116601050 (+1 ms) 2024-12-02T05:16:41,057 INFO [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733116600956 2024-12-02T05:16:41,067 DEBUG [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T05:16:41,068 INFO [RS_OPEN_META-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T05:16:41,069 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:16:41,072 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f49461030c22,46495,1733116594459, state=OPEN 2024-12-02T05:16:41,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T05:16:41,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T05:16:41,257 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T05:16:41,257 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T05:16:41,258 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:41,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T05:16:41,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f49461030c22,46495,1733116594459 in 476 msec 2024-12-02T05:16:41,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T05:16:41,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 922 msec 2024-12-02T05:16:41,275 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T05:16:41,275 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T05:16:41,296 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:16:41,297 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:16:41,316 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:16:41,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41097, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:16:41,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 2.1430 sec 2024-12-02T05:16:41,338 INFO [master/f49461030c22:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733116601338, completionTime=-1 2024-12-02T05:16:41,341 INFO [master/f49461030c22:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-02T05:16:41,341 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T05:16:41,369 INFO [master/f49461030c22:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-02T05:16:41,369 INFO [master/f49461030c22:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733116661369 2024-12-02T05:16:41,369 INFO [master/f49461030c22:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733116721369 2024-12-02T05:16:41,369 INFO [master/f49461030c22:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-12-02T05:16:41,371 INFO [master/f49461030c22:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f49461030c22,42731,1733116593060-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:41,372 INFO [master/f49461030c22:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f49461030c22,42731,1733116593060-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:41,372 INFO [master/f49461030c22:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f49461030c22,42731,1733116593060-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:41,373 INFO [master/f49461030c22:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f49461030c22:42731, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:41,374 INFO [master/f49461030c22:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:41,374 INFO [master/f49461030c22:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:41,380 DEBUG [master/f49461030c22:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T05:16:41,399 INFO [master/f49461030c22:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 6.055sec 2024-12-02T05:16:41,400 INFO [master/f49461030c22:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T05:16:41,401 INFO [master/f49461030c22:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T05:16:41,402 INFO [master/f49461030c22:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T05:16:41,402 INFO [master/f49461030c22:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T05:16:41,402 INFO [master/f49461030c22:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T05:16:41,403 INFO [master/f49461030c22:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f49461030c22,42731,1733116593060-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T05:16:41,403 INFO [master/f49461030c22:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f49461030c22,42731,1733116593060-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T05:16:41,412 DEBUG [master/f49461030c22:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T05:16:41,413 INFO [master/f49461030c22:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T05:16:41,414 INFO [master/f49461030c22:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f49461030c22,42731,1733116593060-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T05:16:41,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:41,445 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-02T05:16:41,446 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-02T05:16:41,448 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:16:41,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:16:41,458 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:16:41,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:16:41,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:16:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b7b2b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:16:41,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:16:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:16:41,470 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:16:41,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:41,473 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:16:41,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:16:41,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:16:41,486 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:16:41,507 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f49461030c22,42731,1733116593060 2024-12-02T05:16:41,523 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=477, ProcessCount=11, AvailableMemoryMB=6579 2024-12-02T05:16:41,531 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T05:16:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f49461030c22,42731,1733116593060 2024-12-02T05:16:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@713ddca7 2024-12-02T05:16:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T05:16:41,543 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T05:16:41,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T05:16:41,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T05:16:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T05:16:41,578 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T05:16:41,580 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-02T05:16:41,581 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:41,583 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T05:16:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T05:16:41,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741835_1011 (size=1061) 2024-12-02T05:16:41,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T05:16:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T05:16:42,016 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c 2024-12-02T05:16:42,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741836_1012 (size=53) 2024-12-02T05:16:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T05:16:42,430 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:16:42,431 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d2d081cdce71d4cc4bebd39e9655d288, disabling compactions & flushes 2024-12-02T05:16:42,431 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:42,431 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:42,431 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. after waiting 0 ms 2024-12-02T05:16:42,431 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:42,431 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:42,432 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d2d081cdce71d4cc4bebd39e9655d288: Waiting for close lock at 1733116602430Disabling compacts and flushes for region at 1733116602430Disabling writes for close at 1733116602431 (+1 ms)Writing region close event to WAL at 1733116602431Closed at 1733116602431 2024-12-02T05:16:42,436 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T05:16:42,443 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733116602436"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733116602436"}]},"ts":"1733116602436"} 2024-12-02T05:16:42,447 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T05:16:42,449 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T05:16:42,451 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116602449"}]},"ts":"1733116602449"} 2024-12-02T05:16:42,455 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T05:16:42,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2d081cdce71d4cc4bebd39e9655d288, ASSIGN}] 2024-12-02T05:16:42,459 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2d081cdce71d4cc4bebd39e9655d288, ASSIGN 2024-12-02T05:16:42,461 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2d081cdce71d4cc4bebd39e9655d288, ASSIGN; state=OFFLINE, location=f49461030c22,46495,1733116594459; forceNewPlan=false, retain=false 2024-12-02T05:16:42,614 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d2d081cdce71d4cc4bebd39e9655d288, regionState=OPENING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:16:42,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2d081cdce71d4cc4bebd39e9655d288, ASSIGN because future has completed 2024-12-02T05:16:42,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:16:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T05:16:42,792 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:42,793 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} 2024-12-02T05:16:42,793 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:42,793 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:16:42,794 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:42,794 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:42,796 INFO [StoreOpener-d2d081cdce71d4cc4bebd39e9655d288-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:42,800 INFO [StoreOpener-d2d081cdce71d4cc4bebd39e9655d288-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:16:42,800 INFO [StoreOpener-d2d081cdce71d4cc4bebd39e9655d288-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2d081cdce71d4cc4bebd39e9655d288 columnFamilyName A 2024-12-02T05:16:42,801 DEBUG [StoreOpener-d2d081cdce71d4cc4bebd39e9655d288-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:42,802 INFO [StoreOpener-d2d081cdce71d4cc4bebd39e9655d288-1 {}] regionserver.HStore(327): Store=d2d081cdce71d4cc4bebd39e9655d288/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:16:42,802 INFO [StoreOpener-d2d081cdce71d4cc4bebd39e9655d288-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:42,804 INFO [StoreOpener-d2d081cdce71d4cc4bebd39e9655d288-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:16:42,805 INFO [StoreOpener-d2d081cdce71d4cc4bebd39e9655d288-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2d081cdce71d4cc4bebd39e9655d288 columnFamilyName B 2024-12-02T05:16:42,805 DEBUG [StoreOpener-d2d081cdce71d4cc4bebd39e9655d288-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:42,806 INFO [StoreOpener-d2d081cdce71d4cc4bebd39e9655d288-1 {}] regionserver.HStore(327): Store=d2d081cdce71d4cc4bebd39e9655d288/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:16:42,807 INFO [StoreOpener-d2d081cdce71d4cc4bebd39e9655d288-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:42,809 INFO [StoreOpener-d2d081cdce71d4cc4bebd39e9655d288-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:16:42,809 INFO [StoreOpener-d2d081cdce71d4cc4bebd39e9655d288-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2d081cdce71d4cc4bebd39e9655d288 columnFamilyName C 2024-12-02T05:16:42,809 DEBUG [StoreOpener-d2d081cdce71d4cc4bebd39e9655d288-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:16:42,810 INFO [StoreOpener-d2d081cdce71d4cc4bebd39e9655d288-1 {}] regionserver.HStore(327): Store=d2d081cdce71d4cc4bebd39e9655d288/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:16:42,811 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:42,811 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:42,812 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:42,813 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:42,814 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:42,815 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:42,815 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T05:16:42,817 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:42,821 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T05:16:42,822 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d2d081cdce71d4cc4bebd39e9655d288; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70896402, jitterRate=0.056438714265823364}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T05:16:42,822 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:42,822 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d2d081cdce71d4cc4bebd39e9655d288: Running coprocessor pre-open hook at 1733116602794Writing region info on filesystem at 1733116602794Initializing all the Stores at 1733116602796 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116602796Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116602796Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116602796Cleaning up temporary data from old regions at 1733116602815 (+19 ms)Running coprocessor post-open hooks at 1733116602822 (+7 ms)Region opened successfully at 1733116602822 2024-12-02T05:16:42,825 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., pid=6, masterSystemTime=1733116602780 2024-12-02T05:16:42,829 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:42,829 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:42,830 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d2d081cdce71d4cc4bebd39e9655d288, regionState=OPEN, openSeqNum=2, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:16:42,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:16:42,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-02T05:16:42,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 in 213 msec 2024-12-02T05:16:42,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-02T05:16:42,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2d081cdce71d4cc4bebd39e9655d288, ASSIGN in 383 msec 2024-12-02T05:16:42,845 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T05:16:42,846 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116602845"}]},"ts":"1733116602845"} 2024-12-02T05:16:42,849 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T05:16:42,851 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T05:16:42,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2820 sec 2024-12-02T05:16:43,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T05:16:43,374 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T05:16:43,385 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T05:16:43,386 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T05:16:43,386 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-02T05:16:43,386 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-02T05:16:43,387 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T05:16:43,387 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-02T05:16:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T05:16:43,752 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:16:43,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:16:43,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:16:43,762 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:16:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:16:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:16:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@508e5ff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:16:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:16:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:16:43,765 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:16:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10fe15c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,767 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:16:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:16:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:16:43,771 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:16:43,773 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T05:16:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f49461030c22,42731,1733116593060 2024-12-02T05:16:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@253960f5 2024-12-02T05:16:43,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T05:16:43,779 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T05:16:43,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,784 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:16:43,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:16:43,786 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:16:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:16:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:16:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0e6697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:16:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:16:43,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:16:43,791 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:16:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec6ad86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:16:43,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:16:43,795 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:16:43,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:16:43,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:16:43,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ac3455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:16:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:16:43,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:16:43,800 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:16:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b0e30d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,802 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:16:43,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:16:43,804 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:16:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:16:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:16:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74cae7c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:16:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:16:43,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:16:43,809 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:16:43,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6573bf69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,812 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:16:43,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:16:43,814 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:16:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:16:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:16:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c0df3b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:16:43,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:16:43,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:16:43,819 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:16:43,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59441750, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:16:43,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:16:43,831 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:16:43,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:16:43,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:16:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677eac6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:16:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:16:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:16:43,834 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:16:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ea0df3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:16:43,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:16:43,838 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:16:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:16:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:16:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf4dbe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:16:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:16:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:16:43,841 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:16:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66d1dde7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,843 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:16:43,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:16:43,844 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:16:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:16:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:16:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aa07968, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:16:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:16:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:16:43,847 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:16:43,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5097bf60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:16:43,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:16:43,852 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:16:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:16:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:16:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12dac1a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:16:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:16:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:16:43,856 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:16:43,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e8d071, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,858 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:16:43,859 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:16:43,860 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:16:43,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:16:43,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:16:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@730a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:16:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:16:43,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:16:43,863 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:16:43,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b32f21b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:16:43,870 DEBUG [Thread-146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:16:43,870 DEBUG [Thread-159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:16:43,870 DEBUG [Thread-144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:16:43,871 DEBUG [Thread-150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:16:43,872 DEBUG [Thread-148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:16:43,873 DEBUG [Thread-161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:16:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:16:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:16:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:16:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:16:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:16:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:16:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:16:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:16:43,875 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:16:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:16:43,875 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:16:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:16:43,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:16:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:16:43,877 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:16:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:16:43,879 DEBUG [Thread-155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:16:43,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:16:43,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:16:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:16:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:16:43,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:16:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:16:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:16:43,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:16:43,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:16:43,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:16:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:16:43,894 DEBUG [Thread-152 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:16:43,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:16:43,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:16:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:16:43,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:16:43,909 DEBUG [Thread-157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:16:43,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:16:43,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:16:43,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:16:43,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:16:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:16:43,935 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:16:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-02T05:16:43,947 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:16:43,955 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:16:43,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:16:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T05:16:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:43,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T05:16:44,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:44,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:44,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:44,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:44,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:44,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T05:16:44,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116664095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116664099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/3e86bad8268a46d38947f761d74a2479 is 50, key is test_row_0/A:col10/1733116603979/Put/seqid=0 2024-12-02T05:16:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116664112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116664117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116664107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:44,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116604083, max timestamp=1733116604104Num uniques 859; 2024-12-02T05:16:44,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116604083, max timestamp=1733116604104Num uniques 859; 2024-12-02T05:16:44,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:16:44,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733116604083, max timestamp=1733116604104Num uniques 859; 2024-12-02T05:16:44,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:16:44,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:16:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:44,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741837_1013 (size=12001) 2024-12-02T05:16:44,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/3e86bad8268a46d38947f761d74a2479 2024-12-02T05:16:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T05:16:44,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/9f81768618b449458ede2c7f9d7b2a41 is 50, key is test_row_0/B:col10/1733116603979/Put/seqid=0 2024-12-02T05:16:44,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:44,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116664329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116664329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:44,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:44,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116664336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741838_1014 (size=12001) 2024-12-02T05:16:44,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116664343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/9f81768618b449458ede2c7f9d7b2a41 2024-12-02T05:16:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116664344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/d6cbed32e7d94883beb6e670c8efff0f is 50, key is test_row_0/C:col10/1733116603979/Put/seqid=0 2024-12-02T05:16:44,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116664448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116664458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116664459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741839_1015 (size=12001) 2024-12-02T05:16:44,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116664470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116664470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:44,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:44,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:44,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:44,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T05:16:44,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116664669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:44,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:44,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116664669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116664679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116664687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116664691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:44,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:44,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:44,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:44,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:44,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/d6cbed32e7d94883beb6e670c8efff0f 2024-12-02T05:16:44,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/3e86bad8268a46d38947f761d74a2479 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/3e86bad8268a46d38947f761d74a2479 2024-12-02T05:16:44,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/3e86bad8268a46d38947f761d74a2479, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T05:16:44,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/9f81768618b449458ede2c7f9d7b2a41 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/9f81768618b449458ede2c7f9d7b2a41 2024-12-02T05:16:44,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/9f81768618b449458ede2c7f9d7b2a41, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T05:16:44,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/d6cbed32e7d94883beb6e670c8efff0f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d6cbed32e7d94883beb6e670c8efff0f 2024-12-02T05:16:44,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d6cbed32e7d94883beb6e670c8efff0f, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T05:16:44,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:44,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:44,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for d2d081cdce71d4cc4bebd39e9655d288 in 1012ms, sequenceid=13, compaction requested=false 2024-12-02T05:16:45,006 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-02T05:16:45,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:45,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:45,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-02T05:16:45,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:45,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:45,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:45,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:45,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:45,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:45,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/c2fb629c636941eea45b18bbb7cba694 is 50, key is test_row_0/A:col10/1733116605010/Put/seqid=0 2024-12-02T05:16:45,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116665039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116665043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116665043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116665046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116665067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741840_1016 (size=12001) 2024-12-02T05:16:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T05:16:45,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:45,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:45,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:45,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:45,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116665179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116665179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116665180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116665189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116665183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:45,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:45,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:45,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:45,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116665398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116665408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116665409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116665409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116665409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/c2fb629c636941eea45b18bbb7cba694 2024-12-02T05:16:45,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:45,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:45,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:45,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:45,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/b9df6c5f613e4a2ca232ae87d309b94b is 50, key is test_row_0/B:col10/1733116605010/Put/seqid=0 2024-12-02T05:16:45,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741841_1017 (size=12001) 2024-12-02T05:16:45,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/b9df6c5f613e4a2ca232ae87d309b94b 2024-12-02T05:16:45,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/43c0846d88c04cf180d7e17432b7d926 is 50, key is test_row_0/C:col10/1733116605010/Put/seqid=0 2024-12-02T05:16:45,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741842_1018 (size=12001) 2024-12-02T05:16:45,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/43c0846d88c04cf180d7e17432b7d926 2024-12-02T05:16:45,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/c2fb629c636941eea45b18bbb7cba694 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/c2fb629c636941eea45b18bbb7cba694 2024-12-02T05:16:45,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:45,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/c2fb629c636941eea45b18bbb7cba694, entries=150, sequenceid=50, filesize=11.7 K 2024-12-02T05:16:45,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/b9df6c5f613e4a2ca232ae87d309b94b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b9df6c5f613e4a2ca232ae87d309b94b 2024-12-02T05:16:45,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b9df6c5f613e4a2ca232ae87d309b94b, entries=150, sequenceid=50, filesize=11.7 K 2024-12-02T05:16:45,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/43c0846d88c04cf180d7e17432b7d926 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/43c0846d88c04cf180d7e17432b7d926 2024-12-02T05:16:45,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/43c0846d88c04cf180d7e17432b7d926, entries=150, sequenceid=50, filesize=11.7 K 2024-12-02T05:16:45,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for d2d081cdce71d4cc4bebd39e9655d288 in 686ms, sequenceid=50, compaction requested=false 2024-12-02T05:16:45,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:45,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T05:16:45,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:45,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:45,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:45,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:45,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:45,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:45,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/fab0afbf8ddb4098bd2bc1eab55d4e12 is 50, key is test_row_0/A:col10/1733116605041/Put/seqid=0 2024-12-02T05:16:45,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741843_1019 (size=12001) 2024-12-02T05:16:45,789 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T05:16:45,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116605763, max timestamp=1733116605794Num uniques 859; 2024-12-02T05:16:45,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:16:45,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116605763, max timestamp=1733116605794Num uniques 859; 2024-12-02T05:16:45,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:16:45,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116605763, max timestamp=1733116605794Num uniques 859; 2024-12-02T05:16:45,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:16:45,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:45,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:45,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:45,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:45,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116665847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116665847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116665849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116665853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116665857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:45,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:45,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:45,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:45,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116665969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116665969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116665969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116665978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:45,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116665979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T05:16:46,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:46,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:46,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:46,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:46,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:46,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:46,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/fab0afbf8ddb4098bd2bc1eab55d4e12 2024-12-02T05:16:46,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116666178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:46,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:46,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/8c1cbcbcd25349c681a61a9a5046b69d is 50, key is test_row_0/B:col10/1733116605041/Put/seqid=0 2024-12-02T05:16:46,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116666190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:46,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116666190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116666190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:46,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116666190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:46,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741844_1020 (size=12001) 2024-12-02T05:16:46,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:46,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:46,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:46,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:46,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:46,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:46,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116666488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:46,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116666508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:46,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116666508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:46,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:46,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116666508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:46,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:46,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116666508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:46,557 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T05:16:46,593 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_203839153_22 at /127.0.0.1:49270 [Receiving block BP-16702269-172.17.0.2-1733116583194:blk_1073741830_1006] {}] datanode.BlockReceiver(464): Slow flushOrSync took 305ms (threshold=300ms), isSync:false, flushTotalNanos=305413709ns, volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/cluster_01f97ee8-23a4-f5a0-8cda-7a5425401614/data/data2/, blockId=1073741830, seqno=99 2024-12-02T05:16:46,594 INFO [AsyncFSWAL-0-hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData-prefix:f49461030c22,42731,1733116593060 {}] wal.AbstractFSWAL(1368): Slow sync cost: 306 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34519,DS-5a5c0e22-a56e-4f32-ae1e-a90082acf4bf,DISK]] 2024-12-02T05:16:46,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/8c1cbcbcd25349c681a61a9a5046b69d 2024-12-02T05:16:46,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/760d1fd9859d4d5daa257a67ca436bc3 is 50, key is test_row_0/C:col10/1733116605041/Put/seqid=0 2024-12-02T05:16:46,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741845_1021 (size=12001) 2024-12-02T05:16:46,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:46,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:46,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:46,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:46,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:46,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:46,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:46,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:46,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:46,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:46,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:46,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:46,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:46,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116666998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:47,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116667027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:47,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:47,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116667027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116667028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:47,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116667028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:47,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:47,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:47,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:47,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/760d1fd9859d4d5daa257a67ca436bc3 2024-12-02T05:16:47,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/fab0afbf8ddb4098bd2bc1eab55d4e12 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/fab0afbf8ddb4098bd2bc1eab55d4e12 2024-12-02T05:16:47,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/fab0afbf8ddb4098bd2bc1eab55d4e12, entries=150, sequenceid=60, filesize=11.7 K 2024-12-02T05:16:47,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/8c1cbcbcd25349c681a61a9a5046b69d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/8c1cbcbcd25349c681a61a9a5046b69d 2024-12-02T05:16:47,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/8c1cbcbcd25349c681a61a9a5046b69d, entries=150, sequenceid=60, filesize=11.7 K 2024-12-02T05:16:47,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/760d1fd9859d4d5daa257a67ca436bc3 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/760d1fd9859d4d5daa257a67ca436bc3 2024-12-02T05:16:47,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/760d1fd9859d4d5daa257a67ca436bc3, entries=150, sequenceid=60, filesize=11.7 K 2024-12-02T05:16:47,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d2d081cdce71d4cc4bebd39e9655d288 in 1405ms, sequenceid=60, compaction requested=true 2024-12-02T05:16:47,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:47,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:16:47,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:47,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:16:47,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:47,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:16:47,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:16:47,146 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:16:47,148 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:16:47,151 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:16:47,152 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/B is initiating minor compaction (all files) 2024-12-02T05:16:47,153 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/B in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:47,155 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/9f81768618b449458ede2c7f9d7b2a41, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b9df6c5f613e4a2ca232ae87d309b94b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/8c1cbcbcd25349c681a61a9a5046b69d] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=35.2 K 2024-12-02T05:16:47,157 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f81768618b449458ede2c7f9d7b2a41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733116603962 2024-12-02T05:16:47,158 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:16:47,158 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/A is initiating minor compaction (all files) 2024-12-02T05:16:47,158 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/A in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:47,158 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/3e86bad8268a46d38947f761d74a2479, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/c2fb629c636941eea45b18bbb7cba694, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/fab0afbf8ddb4098bd2bc1eab55d4e12] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=35.2 K 2024-12-02T05:16:47,159 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting b9df6c5f613e4a2ca232ae87d309b94b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733116604992 2024-12-02T05:16:47,160 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c1cbcbcd25349c681a61a9a5046b69d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733116605037 2024-12-02T05:16:47,160 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e86bad8268a46d38947f761d74a2479, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733116603962 2024-12-02T05:16:47,161 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2fb629c636941eea45b18bbb7cba694, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733116604992 2024-12-02T05:16:47,162 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting fab0afbf8ddb4098bd2bc1eab55d4e12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733116605037 2024-12-02T05:16:47,203 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#A#compaction#9 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:47,205 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/a1677ea89e7846deae68fbfb8dd06658 is 50, key is test_row_0/A:col10/1733116605041/Put/seqid=0 2024-12-02T05:16:47,213 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#B#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:47,214 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/b77c738d2771445a97cb39013f70a8a5 is 50, key is test_row_0/B:col10/1733116605041/Put/seqid=0 2024-12-02T05:16:47,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T05:16:47,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:47,215 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T05:16:47,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:47,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:47,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:47,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:47,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:47,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:47,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/dc26811fa1f44683bc6af87696122fa9 is 50, key is test_row_0/A:col10/1733116605852/Put/seqid=0 2024-12-02T05:16:47,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741846_1022 (size=12104) 2024-12-02T05:16:47,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741847_1023 (size=12104) 2024-12-02T05:16:47,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741848_1024 (size=12001) 2024-12-02T05:16:47,260 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/dc26811fa1f44683bc6af87696122fa9 2024-12-02T05:16:47,270 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/b77c738d2771445a97cb39013f70a8a5 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b77c738d2771445a97cb39013f70a8a5 2024-12-02T05:16:47,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/7b78951087ea43f685ad4164b3cc26e9 is 50, key is test_row_0/B:col10/1733116605852/Put/seqid=0 2024-12-02T05:16:47,313 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/B of d2d081cdce71d4cc4bebd39e9655d288 into b77c738d2771445a97cb39013f70a8a5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:47,313 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:47,316 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/B, priority=13, startTime=1733116607144; duration=0sec 2024-12-02T05:16:47,318 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:16:47,318 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:B 2024-12-02T05:16:47,318 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:16:47,320 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:16:47,321 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/C is initiating minor compaction (all files) 2024-12-02T05:16:47,321 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/C in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:47,321 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d6cbed32e7d94883beb6e670c8efff0f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/43c0846d88c04cf180d7e17432b7d926, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/760d1fd9859d4d5daa257a67ca436bc3] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=35.2 K 2024-12-02T05:16:47,322 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting d6cbed32e7d94883beb6e670c8efff0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733116603962 2024-12-02T05:16:47,323 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 43c0846d88c04cf180d7e17432b7d926, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733116604992 2024-12-02T05:16:47,324 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 760d1fd9859d4d5daa257a67ca436bc3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733116605037 2024-12-02T05:16:47,352 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#C#compaction#13 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:47,353 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/5481fbbf5e224c96bfc569098859753b is 50, key is test_row_0/C:col10/1733116605041/Put/seqid=0 2024-12-02T05:16:47,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741849_1025 (size=12001) 2024-12-02T05:16:47,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741850_1026 (size=12104) 2024-12-02T05:16:47,413 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/5481fbbf5e224c96bfc569098859753b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5481fbbf5e224c96bfc569098859753b 2024-12-02T05:16:47,429 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/C of d2d081cdce71d4cc4bebd39e9655d288 into 5481fbbf5e224c96bfc569098859753b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:47,430 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:47,430 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/C, priority=13, startTime=1733116607145; duration=0sec 2024-12-02T05:16:47,430 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:47,430 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:C 2024-12-02T05:16:47,662 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/a1677ea89e7846deae68fbfb8dd06658 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/a1677ea89e7846deae68fbfb8dd06658 2024-12-02T05:16:47,679 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/A of d2d081cdce71d4cc4bebd39e9655d288 into a1677ea89e7846deae68fbfb8dd06658(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:47,679 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:47,680 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/A, priority=13, startTime=1733116607132; duration=0sec 2024-12-02T05:16:47,680 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:47,680 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:A 2024-12-02T05:16:47,757 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/7b78951087ea43f685ad4164b3cc26e9 2024-12-02T05:16:47,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/d48089a3d10c41fc8b56df25200ce783 is 50, key is test_row_0/C:col10/1733116605852/Put/seqid=0 2024-12-02T05:16:47,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741851_1027 (size=12001) 2024-12-02T05:16:47,826 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/d48089a3d10c41fc8b56df25200ce783 2024-12-02T05:16:47,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/dc26811fa1f44683bc6af87696122fa9 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/dc26811fa1f44683bc6af87696122fa9 2024-12-02T05:16:47,858 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/dc26811fa1f44683bc6af87696122fa9, entries=150, sequenceid=97, filesize=11.7 K 2024-12-02T05:16:47,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/7b78951087ea43f685ad4164b3cc26e9 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/7b78951087ea43f685ad4164b3cc26e9 2024-12-02T05:16:47,879 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/7b78951087ea43f685ad4164b3cc26e9, entries=150, sequenceid=97, filesize=11.7 K 2024-12-02T05:16:47,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/d48089a3d10c41fc8b56df25200ce783 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d48089a3d10c41fc8b56df25200ce783 2024-12-02T05:16:47,894 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d48089a3d10c41fc8b56df25200ce783, entries=150, sequenceid=97, filesize=11.7 K 2024-12-02T05:16:47,896 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for d2d081cdce71d4cc4bebd39e9655d288 in 681ms, sequenceid=97, compaction requested=false 2024-12-02T05:16:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-02T05:16:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-02T05:16:47,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-02T05:16:47,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9430 sec 2024-12-02T05:16:47,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.9700 sec 2024-12-02T05:16:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:48,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T05:16:48,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:48,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:48,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:48,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:48,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:48,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T05:16:48,119 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:16:48,119 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:16:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-02T05:16:48,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/79ae191becae493cb17ef12b942371ff is 50, key is test_row_0/A:col10/1733116608057/Put/seqid=0 2024-12-02T05:16:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T05:16:48,126 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:16:48,128 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:16:48,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:16:48,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741852_1028 (size=12001) 2024-12-02T05:16:48,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/79ae191becae493cb17ef12b942371ff 2024-12-02T05:16:48,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733116608149, max timestamp=1733116608171Num uniques 859; 2024-12-02T05:16:48,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:16:48,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116608149, max timestamp=1733116608171Num uniques 859; 2024-12-02T05:16:48,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/cb1860a05fec4aaba727da585a63f277 is 50, key is test_row_0/B:col10/1733116608057/Put/seqid=0 2024-12-02T05:16:48,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:16:48,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733116608160, max timestamp=1733116608178Num uniques 859; 2024-12-02T05:16:48,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T05:16:48,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741853_1029 (size=12001) 2024-12-02T05:16:48,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/cb1860a05fec4aaba727da585a63f277 2024-12-02T05:16:48,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/9ddb7c6a74de479d91cf96ee2e98cb09 is 50, key is test_row_0/C:col10/1733116608057/Put/seqid=0 2024-12-02T05:16:48,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116668215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116668216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116668215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T05:16:48,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116668212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116668221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741854_1030 (size=12001) 2024-12-02T05:16:48,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/9ddb7c6a74de479d91cf96ee2e98cb09 2024-12-02T05:16:48,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/79ae191becae493cb17ef12b942371ff as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/79ae191becae493cb17ef12b942371ff 2024-12-02T05:16:48,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/79ae191becae493cb17ef12b942371ff, entries=150, sequenceid=111, filesize=11.7 K 2024-12-02T05:16:48,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/cb1860a05fec4aaba727da585a63f277 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/cb1860a05fec4aaba727da585a63f277 2024-12-02T05:16:48,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T05:16:48,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:48,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:48,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:48,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:48,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:48,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/cb1860a05fec4aaba727da585a63f277, entries=150, sequenceid=111, filesize=11.7 K 2024-12-02T05:16:48,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/9ddb7c6a74de479d91cf96ee2e98cb09 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9ddb7c6a74de479d91cf96ee2e98cb09 2024-12-02T05:16:48,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9ddb7c6a74de479d91cf96ee2e98cb09, entries=150, sequenceid=111, filesize=11.7 K 2024-12-02T05:16:48,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=124.84 KB/127835 for d2d081cdce71d4cc4bebd39e9655d288 in 214ms, sequenceid=111, compaction requested=true 2024-12-02T05:16:48,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:48,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:16:48,326 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:16:48,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:48,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:16:48,326 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:16:48,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:48,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:16:48,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:16:48,328 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:16:48,329 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/B is initiating minor compaction (all files) 2024-12-02T05:16:48,329 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/B in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:48,329 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b77c738d2771445a97cb39013f70a8a5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/7b78951087ea43f685ad4164b3cc26e9, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/cb1860a05fec4aaba727da585a63f277] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=35.3 K 2024-12-02T05:16:48,330 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:16:48,331 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/A is initiating minor compaction (all files) 2024-12-02T05:16:48,331 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/A in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:48,331 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/a1677ea89e7846deae68fbfb8dd06658, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/dc26811fa1f44683bc6af87696122fa9, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/79ae191becae493cb17ef12b942371ff] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=35.3 K 2024-12-02T05:16:48,334 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting b77c738d2771445a97cb39013f70a8a5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733116605037 2024-12-02T05:16:48,335 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1677ea89e7846deae68fbfb8dd06658, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733116605037 2024-12-02T05:16:48,336 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b78951087ea43f685ad4164b3cc26e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733116605841 2024-12-02T05:16:48,337 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc26811fa1f44683bc6af87696122fa9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733116605841 2024-12-02T05:16:48,339 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79ae191becae493cb17ef12b942371ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733116608054 2024-12-02T05:16:48,340 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting cb1860a05fec4aaba727da585a63f277, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733116608054 2024-12-02T05:16:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:48,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=131.55 KB heapSize=374.30 KB 2024-12-02T05:16:48,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:48,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:48,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:48,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:48,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:48,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:48,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116668352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116668352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116668356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/f1b2ed3e9a34428097a1d73ffe5ad0fa is 50, key is test_row_0/A:col10/1733116608205/Put/seqid=0 2024-12-02T05:16:48,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116668361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116668364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,390 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#B#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:48,391 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/144c1e4c1cde4117918facadf33cf626 is 50, key is test_row_0/B:col10/1733116608057/Put/seqid=0 2024-12-02T05:16:48,395 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#A#compaction#20 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:48,396 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/7aa9408897ab415f922e671853f6f5de is 50, key is test_row_0/A:col10/1733116608057/Put/seqid=0 2024-12-02T05:16:48,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741856_1032 (size=12207) 2024-12-02T05:16:48,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741855_1031 (size=14541) 2024-12-02T05:16:48,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741857_1033 (size=12207) 2024-12-02T05:16:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T05:16:48,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T05:16:48,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:48,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:48,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:48,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:48,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:48,453 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/7aa9408897ab415f922e671853f6f5de as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/7aa9408897ab415f922e671853f6f5de 2024-12-02T05:16:48,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116668467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,469 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/A of d2d081cdce71d4cc4bebd39e9655d288 into 7aa9408897ab415f922e671853f6f5de(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:48,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116668468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,469 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:48,469 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/A, priority=13, startTime=1733116608326; duration=0sec 2024-12-02T05:16:48,470 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:16:48,470 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:A 2024-12-02T05:16:48,470 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:16:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,472 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:16:48,473 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/C is initiating minor compaction (all files) 2024-12-02T05:16:48,473 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/C in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:48,474 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5481fbbf5e224c96bfc569098859753b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d48089a3d10c41fc8b56df25200ce783, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9ddb7c6a74de479d91cf96ee2e98cb09] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=35.3 K 2024-12-02T05:16:48,476 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5481fbbf5e224c96bfc569098859753b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733116605037 2024-12-02T05:16:48,477 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting d48089a3d10c41fc8b56df25200ce783, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733116605841 2024-12-02T05:16:48,478 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ddb7c6a74de479d91cf96ee2e98cb09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733116608054 2024-12-02T05:16:48,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116668478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116668478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116668478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,498 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#C#compaction#21 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:48,499 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/0b8c98a8e8a643e6939de4bad55a37f2 is 50, key is test_row_0/C:col10/1733116608057/Put/seqid=0 2024-12-02T05:16:48,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741858_1034 (size=12207) 2024-12-02T05:16:48,554 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/0b8c98a8e8a643e6939de4bad55a37f2 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/0b8c98a8e8a643e6939de4bad55a37f2 2024-12-02T05:16:48,577 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/C of d2d081cdce71d4cc4bebd39e9655d288 into 0b8c98a8e8a643e6939de4bad55a37f2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:48,577 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:48,578 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/C, priority=13, startTime=1733116608326; duration=0sec 2024-12-02T05:16:48,579 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:48,579 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:C 2024-12-02T05:16:48,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T05:16:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:48,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:48,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:48,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:48,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:48,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116668677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116668678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116668688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116668688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116668698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T05:16:48,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T05:16:48,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:48,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:48,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:48,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:48,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:48,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:48,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/f1b2ed3e9a34428097a1d73ffe5ad0fa 2024-12-02T05:16:48,841 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/144c1e4c1cde4117918facadf33cf626 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/144c1e4c1cde4117918facadf33cf626 2024-12-02T05:16:48,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/5bef9c19cd954ccaba8ad97b47134b3b is 50, key is test_row_0/B:col10/1733116608205/Put/seqid=0 2024-12-02T05:16:48,855 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/B of d2d081cdce71d4cc4bebd39e9655d288 into 144c1e4c1cde4117918facadf33cf626(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:48,855 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:48,856 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/B, priority=13, startTime=1733116608326; duration=0sec 2024-12-02T05:16:48,857 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:48,857 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:B 2024-12-02T05:16:48,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741859_1035 (size=12151) 2024-12-02T05:16:48,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/5bef9c19cd954ccaba8ad97b47134b3b 2024-12-02T05:16:48,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T05:16:48,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:48,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:48,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:48,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:48,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:48,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/1cc4069eff3247dd8d4c2a0309f0dd9c is 50, key is test_row_0/C:col10/1733116608205/Put/seqid=0 2024-12-02T05:16:48,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741860_1036 (size=12151) 2024-12-02T05:16:48,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/1cc4069eff3247dd8d4c2a0309f0dd9c 2024-12-02T05:16:48,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116668988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/f1b2ed3e9a34428097a1d73ffe5ad0fa as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f1b2ed3e9a34428097a1d73ffe5ad0fa 2024-12-02T05:16:48,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116668988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116668999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:48,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116668998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f1b2ed3e9a34428097a1d73ffe5ad0fa, entries=200, sequenceid=147, filesize=14.2 K 2024-12-02T05:16:49,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/5bef9c19cd954ccaba8ad97b47134b3b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/5bef9c19cd954ccaba8ad97b47134b3b 2024-12-02T05:16:49,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/5bef9c19cd954ccaba8ad97b47134b3b, entries=150, sequenceid=147, filesize=11.9 K 2024-12-02T05:16:49,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116669018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/1cc4069eff3247dd8d4c2a0309f0dd9c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/1cc4069eff3247dd8d4c2a0309f0dd9c 2024-12-02T05:16:49,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/1cc4069eff3247dd8d4c2a0309f0dd9c, entries=150, sequenceid=147, filesize=11.9 K 2024-12-02T05:16:49,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.55 KB/134705, heapSize ~374.26 KB/383240, currentSize=46.96 KB/48090 for d2d081cdce71d4cc4bebd39e9655d288 in 711ms, sequenceid=147, compaction requested=false 2024-12-02T05:16:49,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:49,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T05:16:49,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:49,065 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T05:16:49,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:49,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/8b84a59768c14bc9880ab6a697231f7b is 50, key is test_row_1/A:col10/1733116608357/Put/seqid=0 2024-12-02T05:16:49,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741861_1037 (size=9757) 2024-12-02T05:16:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T05:16:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:49,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:49,517 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/8b84a59768c14bc9880ab6a697231f7b 2024-12-02T05:16:49,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/b9393e79317e46b7a480192ee37ecd33 is 50, key is test_row_1/B:col10/1733116608357/Put/seqid=0 2024-12-02T05:16:49,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116609532, max timestamp=1733116609541Num uniques 859; 2024-12-02T05:16:49,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:16:49,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116609532, max timestamp=1733116609541Num uniques 859; 2024-12-02T05:16:49,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:16:49,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116609532, max timestamp=1733116609541Num uniques 859; 2024-12-02T05:16:49,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:16:49,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741862_1038 (size=9757) 2024-12-02T05:16:49,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116669574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116669576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116669577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116669580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116669583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116669688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116669688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116669688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116669688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116669698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116669897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116669897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116669898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116669898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116669907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:49,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:49,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:49,969 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/b9393e79317e46b7a480192ee37ecd33 2024-12-02T05:16:49,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/61dc9ed247c24f74897c73c19b0a58d5 is 50, key is test_row_1/C:col10/1733116608357/Put/seqid=0 2024-12-02T05:16:49,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741863_1039 (size=9757) 2024-12-02T05:16:50,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:50,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116670208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:50,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:50,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:50,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116670209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:50,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116670209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:50,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116670209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:50,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:50,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:50,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116670218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:50,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T05:16:50,393 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/61dc9ed247c24f74897c73c19b0a58d5 2024-12-02T05:16:50,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/8b84a59768c14bc9880ab6a697231f7b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/8b84a59768c14bc9880ab6a697231f7b 2024-12-02T05:16:50,429 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/8b84a59768c14bc9880ab6a697231f7b, entries=100, sequenceid=160, filesize=9.5 K 2024-12-02T05:16:50,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/b9393e79317e46b7a480192ee37ecd33 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b9393e79317e46b7a480192ee37ecd33 2024-12-02T05:16:50,453 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b9393e79317e46b7a480192ee37ecd33, entries=100, sequenceid=160, filesize=9.5 K 2024-12-02T05:16:50,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/61dc9ed247c24f74897c73c19b0a58d5 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/61dc9ed247c24f74897c73c19b0a58d5 2024-12-02T05:16:50,472 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/61dc9ed247c24f74897c73c19b0a58d5, entries=100, sequenceid=160, filesize=9.5 K 2024-12-02T05:16:50,473 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d2d081cdce71d4cc4bebd39e9655d288 in 1408ms, sequenceid=160, compaction requested=true 2024-12-02T05:16:50,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:50,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:50,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-02T05:16:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-02T05:16:50,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-02T05:16:50,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3480 sec 2024-12-02T05:16:50,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.3630 sec 2024-12-02T05:16:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:50,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-02T05:16:50,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:50,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T05:16:50,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:50,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T05:16:50,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:50,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T05:16:50,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/c31abbbd27e94fa0ade31b7665e721f4 is 50, key is test_row_0/A:col10/1733116610718/Put/seqid=0 2024-12-02T05:16:50,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116670733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:50,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116670734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:50,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:50,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116670734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:50,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:50,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116670736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116670736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:50,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:50,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741864_1040 (size=12151) 2024-12-02T05:16:50,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/c31abbbd27e94fa0ade31b7665e721f4 2024-12-02T05:16:50,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/378ab6b983eb43abace43a9481bf2793 is 50, key is test_row_0/B:col10/1733116610718/Put/seqid=0 2024-12-02T05:16:50,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741865_1041 (size=12151) 2024-12-02T05:16:50,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/378ab6b983eb43abace43a9481bf2793 2024-12-02T05:16:50,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:50,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116670847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116670847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:50,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:50,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116670848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116670848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:50,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116670848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:50,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/7bbfacceddc04014ac569c45aec895de is 50, key is test_row_0/C:col10/1733116610718/Put/seqid=0 2024-12-02T05:16:50,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741866_1042 (size=12151) 2024-12-02T05:16:51,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116671058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116671058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116671058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116671058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116671068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/7bbfacceddc04014ac569c45aec895de 2024-12-02T05:16:51,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/c31abbbd27e94fa0ade31b7665e721f4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/c31abbbd27e94fa0ade31b7665e721f4 2024-12-02T05:16:51,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/c31abbbd27e94fa0ade31b7665e721f4, entries=150, sequenceid=198, filesize=11.9 K 2024-12-02T05:16:51,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/378ab6b983eb43abace43a9481bf2793 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/378ab6b983eb43abace43a9481bf2793 2024-12-02T05:16:51,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116671368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116671368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116671368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116671369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/378ab6b983eb43abace43a9481bf2793, entries=150, sequenceid=198, filesize=11.9 K 2024-12-02T05:16:51,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/7bbfacceddc04014ac569c45aec895de as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/7bbfacceddc04014ac569c45aec895de 2024-12-02T05:16:51,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/7bbfacceddc04014ac569c45aec895de, entries=150, sequenceid=198, filesize=11.9 K 2024-12-02T05:16:51,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=46.96 KB/48090 for d2d081cdce71d4cc4bebd39e9655d288 in 665ms, sequenceid=198, compaction requested=true 2024-12-02T05:16:51,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:51,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:16:51,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:51,386 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:16:51,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:16:51,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:16:51,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:16:51,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:16:51,386 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:16:51,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T05:16:51,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:51,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:51,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:51,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:51,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:51,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:51,390 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:16:51,390 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/B is initiating minor compaction (all files) 2024-12-02T05:16:51,390 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/B in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:51,391 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/144c1e4c1cde4117918facadf33cf626, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/5bef9c19cd954ccaba8ad97b47134b3b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b9393e79317e46b7a480192ee37ecd33, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/378ab6b983eb43abace43a9481bf2793] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=45.2 K 2024-12-02T05:16:51,391 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48656 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:16:51,391 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/A is initiating minor compaction (all files) 2024-12-02T05:16:51,391 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/A in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:51,391 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/7aa9408897ab415f922e671853f6f5de, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f1b2ed3e9a34428097a1d73ffe5ad0fa, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/8b84a59768c14bc9880ab6a697231f7b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/c31abbbd27e94fa0ade31b7665e721f4] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=47.5 K 2024-12-02T05:16:51,392 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 144c1e4c1cde4117918facadf33cf626, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733116608054 2024-12-02T05:16:51,393 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aa9408897ab415f922e671853f6f5de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733116608054 2024-12-02T05:16:51,394 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bef9c19cd954ccaba8ad97b47134b3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733116608205 2024-12-02T05:16:51,396 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1b2ed3e9a34428097a1d73ffe5ad0fa, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733116608205 2024-12-02T05:16:51,397 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting b9393e79317e46b7a480192ee37ecd33, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733116608357 2024-12-02T05:16:51,397 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b84a59768c14bc9880ab6a697231f7b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733116608357 2024-12-02T05:16:51,399 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 378ab6b983eb43abace43a9481bf2793, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733116609576 2024-12-02T05:16:51,401 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting c31abbbd27e94fa0ade31b7665e721f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733116609576 2024-12-02T05:16:51,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/7fc33ff89037455dacd0e8cac1f47934 is 50, key is test_row_0/A:col10/1733116610733/Put/seqid=0 2024-12-02T05:16:51,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741867_1043 (size=12151) 2024-12-02T05:16:51,438 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:51,442 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/ff8077f01f37472aae6a2b47234737b6 is 50, key is test_row_0/B:col10/1733116610718/Put/seqid=0 2024-12-02T05:16:51,447 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#A#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:51,448 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/51a8c32b23ff4172a386f7fce00e7d53 is 50, key is test_row_0/A:col10/1733116610718/Put/seqid=0 2024-12-02T05:16:51,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741869_1045 (size=12493) 2024-12-02T05:16:51,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741868_1044 (size=12493) 2024-12-02T05:16:51,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116611447, max timestamp=1733116611510Num uniques 859; 2024-12-02T05:16:51,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116611447, max timestamp=1733116611510Num uniques 859; 2024-12-02T05:16:51,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:16:51,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116611447, max timestamp=1733116611510Num uniques 859; 2024-12-02T05:16:51,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:16:51,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:16:51,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116671608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116671718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/7fc33ff89037455dacd0e8cac1f47934 2024-12-02T05:16:51,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/52abf69fcc334fe894e765756be56c4d is 50, key is test_row_0/B:col10/1733116610733/Put/seqid=0 2024-12-02T05:16:51,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741870_1046 (size=12151) 2024-12-02T05:16:51,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116671878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116671878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116671879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116671879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,891 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/51a8c32b23ff4172a386f7fce00e7d53 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/51a8c32b23ff4172a386f7fce00e7d53 2024-12-02T05:16:51,896 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/ff8077f01f37472aae6a2b47234737b6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/ff8077f01f37472aae6a2b47234737b6 2024-12-02T05:16:51,903 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/A of d2d081cdce71d4cc4bebd39e9655d288 into 51a8c32b23ff4172a386f7fce00e7d53(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:51,903 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:51,903 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/A, priority=12, startTime=1733116611386; duration=0sec 2024-12-02T05:16:51,903 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:16:51,903 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:A 2024-12-02T05:16:51,903 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:16:51,905 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:16:51,905 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/C is initiating minor compaction (all files) 2024-12-02T05:16:51,905 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/C in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:51,906 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/0b8c98a8e8a643e6939de4bad55a37f2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/1cc4069eff3247dd8d4c2a0309f0dd9c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/61dc9ed247c24f74897c73c19b0a58d5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/7bbfacceddc04014ac569c45aec895de] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=45.2 K 2024-12-02T05:16:51,906 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/B of d2d081cdce71d4cc4bebd39e9655d288 into ff8077f01f37472aae6a2b47234737b6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:51,906 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b8c98a8e8a643e6939de4bad55a37f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733116608054 2024-12-02T05:16:51,906 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:51,907 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/B, priority=12, startTime=1733116611386; duration=0sec 2024-12-02T05:16:51,907 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:51,907 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:B 2024-12-02T05:16:51,908 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cc4069eff3247dd8d4c2a0309f0dd9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733116608205 2024-12-02T05:16:51,908 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61dc9ed247c24f74897c73c19b0a58d5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733116608357 2024-12-02T05:16:51,909 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bbfacceddc04014ac569c45aec895de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733116609576 2024-12-02T05:16:51,925 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#C#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:51,926 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/08fc7e64b4b848e297b738ed4f114569 is 50, key is test_row_0/C:col10/1733116610718/Put/seqid=0 2024-12-02T05:16:51,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116671928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:51,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741871_1047 (size=12493) 2024-12-02T05:16:51,954 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/08fc7e64b4b848e297b738ed4f114569 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/08fc7e64b4b848e297b738ed4f114569 2024-12-02T05:16:51,963 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/C of d2d081cdce71d4cc4bebd39e9655d288 into 08fc7e64b4b848e297b738ed4f114569(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:51,963 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:51,963 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/C, priority=12, startTime=1733116611386; duration=0sec 2024-12-02T05:16:51,963 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:51,963 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:C 2024-12-02T05:16:52,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116672237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:52,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/52abf69fcc334fe894e765756be56c4d 2024-12-02T05:16:52,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/18f13124437c40a0ae07b623abedb367 is 50, key is test_row_0/C:col10/1733116610733/Put/seqid=0 2024-12-02T05:16:52,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741872_1048 (size=12151) 2024-12-02T05:16:52,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/18f13124437c40a0ae07b623abedb367 2024-12-02T05:16:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T05:16:52,298 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:16:52,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:16:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-02T05:16:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T05:16:52,302 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:16:52,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/7fc33ff89037455dacd0e8cac1f47934 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/7fc33ff89037455dacd0e8cac1f47934 2024-12-02T05:16:52,304 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:16:52,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:16:52,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/7fc33ff89037455dacd0e8cac1f47934, entries=150, sequenceid=209, filesize=11.9 K 2024-12-02T05:16:52,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/52abf69fcc334fe894e765756be56c4d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/52abf69fcc334fe894e765756be56c4d 2024-12-02T05:16:52,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/52abf69fcc334fe894e765756be56c4d, entries=150, sequenceid=209, filesize=11.9 K 2024-12-02T05:16:52,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/18f13124437c40a0ae07b623abedb367 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/18f13124437c40a0ae07b623abedb367 2024-12-02T05:16:52,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/18f13124437c40a0ae07b623abedb367, entries=150, sequenceid=209, filesize=11.9 K 2024-12-02T05:16:52,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d2d081cdce71d4cc4bebd39e9655d288 in 959ms, sequenceid=209, compaction requested=false 2024-12-02T05:16:52,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T05:16:52,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T05:16:52,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:52,458 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-02T05:16:52,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:52,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:52,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:52,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:52,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:52,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:52,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/084bfbdecc864749ae061f35a22ac02a is 50, key is test_row_0/A:col10/1733116611574/Put/seqid=0 2024-12-02T05:16:52,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741873_1049 (size=12151) 2024-12-02T05:16:52,473 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/084bfbdecc864749ae061f35a22ac02a 2024-12-02T05:16:52,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/0a219f307b334220a02d6cc35c7b9376 is 50, key is test_row_0/B:col10/1733116611574/Put/seqid=0 2024-12-02T05:16:52,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741874_1050 (size=12151) 2024-12-02T05:16:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T05:16:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:52,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:52,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116672796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:52,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:52,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:52,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:52,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116672887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:52,892 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in d2d081cdce71d4cc4bebd39e9655d288 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:16:52,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:52,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:52,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:52,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116672898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:52,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116672899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:52,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:52,902 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/0a219f307b334220a02d6cc35c7b9376 2024-12-02T05:16:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116672899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:52,901 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in d2d081cdce71d4cc4bebd39e9655d288 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:16:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:52,903 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in d2d081cdce71d4cc4bebd39e9655d288 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:16:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:52,902 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in d2d081cdce71d4cc4bebd39e9655d288 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:16:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:52,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116672909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:52,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:52,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:52,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/a4afa3593c5c41e9aad9a02efe26734d is 50, key is test_row_0/C:col10/1733116611574/Put/seqid=0 2024-12-02T05:16:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T05:16:52,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741875_1051 (size=12151) 2024-12-02T05:16:53,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116673117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:53,343 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/a4afa3593c5c41e9aad9a02efe26734d 2024-12-02T05:16:53,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/084bfbdecc864749ae061f35a22ac02a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/084bfbdecc864749ae061f35a22ac02a 2024-12-02T05:16:53,361 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/084bfbdecc864749ae061f35a22ac02a, entries=150, sequenceid=247, filesize=11.9 K 2024-12-02T05:16:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/0a219f307b334220a02d6cc35c7b9376 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0a219f307b334220a02d6cc35c7b9376 2024-12-02T05:16:53,372 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0a219f307b334220a02d6cc35c7b9376, entries=150, sequenceid=247, filesize=11.9 K 2024-12-02T05:16:53,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/a4afa3593c5c41e9aad9a02efe26734d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/a4afa3593c5c41e9aad9a02efe26734d 2024-12-02T05:16:53,386 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/a4afa3593c5c41e9aad9a02efe26734d, entries=150, sequenceid=247, filesize=11.9 K 2024-12-02T05:16:53,388 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for d2d081cdce71d4cc4bebd39e9655d288 in 930ms, sequenceid=247, compaction requested=true 2024-12-02T05:16:53,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-02T05:16:53,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-02T05:16:53,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-02T05:16:53,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0880 sec 2024-12-02T05:16:53,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.0970 sec 2024-12-02T05:16:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:53,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T05:16:53,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:53,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:53,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:53,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:53,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:53,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T05:16:53,438 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:16:53,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/5d95780d5f9f480d89adfa56f99d1d14 is 50, key is test_row_0/A:col10/1733116612779/Put/seqid=0 2024-12-02T05:16:53,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:16:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-02T05:16:53,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T05:16:53,444 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:16:53,446 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:16:53,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:16:53,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741876_1052 (size=14641) 2024-12-02T05:16:53,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/5d95780d5f9f480d89adfa56f99d1d14 2024-12-02T05:16:53,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/02344fb6a52f4d908356128e9f824691 is 50, key is test_row_0/B:col10/1733116612779/Put/seqid=0 2024-12-02T05:16:53,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741877_1053 (size=12201) 2024-12-02T05:16:53,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/02344fb6a52f4d908356128e9f824691 2024-12-02T05:16:53,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116673533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:53,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/660158e951014a25923d9f7bc71dbdc4 is 50, key is test_row_0/C:col10/1733116612779/Put/seqid=0 2024-12-02T05:16:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T05:16:53,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741878_1054 (size=12201) 2024-12-02T05:16:53,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T05:16:53,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:53,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:53,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:53,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:53,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:53,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116673647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:53,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T05:16:53,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:53,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:53,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:53,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:53,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T05:16:53,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116673857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:53,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T05:16:53,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:53,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:53,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:53,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:53,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:53,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/660158e951014a25923d9f7bc71dbdc4 2024-12-02T05:16:53,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/5d95780d5f9f480d89adfa56f99d1d14 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/5d95780d5f9f480d89adfa56f99d1d14 2024-12-02T05:16:53,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/5d95780d5f9f480d89adfa56f99d1d14, entries=200, sequenceid=259, filesize=14.3 K 2024-12-02T05:16:53,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/02344fb6a52f4d908356128e9f824691 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/02344fb6a52f4d908356128e9f824691 2024-12-02T05:16:53,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/02344fb6a52f4d908356128e9f824691, entries=150, sequenceid=259, filesize=11.9 K 2024-12-02T05:16:53,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/660158e951014a25923d9f7bc71dbdc4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/660158e951014a25923d9f7bc71dbdc4 2024-12-02T05:16:53,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/660158e951014a25923d9f7bc71dbdc4, entries=150, sequenceid=259, filesize=11.9 K 2024-12-02T05:16:53,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d2d081cdce71d4cc4bebd39e9655d288 in 555ms, sequenceid=259, compaction requested=true 2024-12-02T05:16:53,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:53,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:16:53,987 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:16:53,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:53,987 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:16:53,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:16:53,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:53,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:16:53,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:16:53,989 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48996 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:16:53,989 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51436 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:16:53,989 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/B is initiating minor compaction (all files) 2024-12-02T05:16:53,989 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/A is initiating minor compaction (all files) 2024-12-02T05:16:53,989 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/A in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:53,989 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/B in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:53,990 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/51a8c32b23ff4172a386f7fce00e7d53, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/7fc33ff89037455dacd0e8cac1f47934, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/084bfbdecc864749ae061f35a22ac02a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/5d95780d5f9f480d89adfa56f99d1d14] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=50.2 K 2024-12-02T05:16:53,990 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/ff8077f01f37472aae6a2b47234737b6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/52abf69fcc334fe894e765756be56c4d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0a219f307b334220a02d6cc35c7b9376, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/02344fb6a52f4d908356128e9f824691] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=47.8 K 2024-12-02T05:16:53,990 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting ff8077f01f37472aae6a2b47234737b6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733116609576 2024-12-02T05:16:53,990 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51a8c32b23ff4172a386f7fce00e7d53, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733116609576 2024-12-02T05:16:53,991 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fc33ff89037455dacd0e8cac1f47934, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733116610733 2024-12-02T05:16:53,991 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 52abf69fcc334fe894e765756be56c4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733116610733 2024-12-02T05:16:53,991 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 084bfbdecc864749ae061f35a22ac02a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733116611574 2024-12-02T05:16:53,991 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a219f307b334220a02d6cc35c7b9376, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733116611574 2024-12-02T05:16:53,992 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d95780d5f9f480d89adfa56f99d1d14, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733116612774 2024-12-02T05:16:53,992 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 02344fb6a52f4d908356128e9f824691, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733116612774 2024-12-02T05:16:54,007 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:54,007 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/6b65257b4c1b45f0a586bc52a06ec8b1 is 50, key is test_row_0/A:col10/1733116612779/Put/seqid=0 2024-12-02T05:16:54,015 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:54,016 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/3670379f92f34703a4c5d9fe4e9eaaf1 is 50, key is test_row_0/B:col10/1733116612779/Put/seqid=0 2024-12-02T05:16:54,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741879_1055 (size=12679) 2024-12-02T05:16:54,055 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/6b65257b4c1b45f0a586bc52a06ec8b1 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/6b65257b4c1b45f0a586bc52a06ec8b1 2024-12-02T05:16:54,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741880_1056 (size=12679) 2024-12-02T05:16:54,067 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/A of d2d081cdce71d4cc4bebd39e9655d288 into 6b65257b4c1b45f0a586bc52a06ec8b1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:54,067 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:54,067 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/A, priority=12, startTime=1733116613987; duration=0sec 2024-12-02T05:16:54,067 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:16:54,067 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:A 2024-12-02T05:16:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T05:16:54,067 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:16:54,073 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48996 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:16:54,074 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/C is initiating minor compaction (all files) 2024-12-02T05:16:54,074 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/C in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:54,074 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/08fc7e64b4b848e297b738ed4f114569, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/18f13124437c40a0ae07b623abedb367, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/a4afa3593c5c41e9aad9a02efe26734d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/660158e951014a25923d9f7bc71dbdc4] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=47.8 K 2024-12-02T05:16:54,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T05:16:54,076 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08fc7e64b4b848e297b738ed4f114569, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733116609576 2024-12-02T05:16:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:54,076 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/3670379f92f34703a4c5d9fe4e9eaaf1 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/3670379f92f34703a4c5d9fe4e9eaaf1 2024-12-02T05:16:54,076 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T05:16:54,076 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18f13124437c40a0ae07b623abedb367, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733116610733 2024-12-02T05:16:54,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:54,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:54,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:54,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:54,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:54,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:54,078 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4afa3593c5c41e9aad9a02efe26734d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733116611574 2024-12-02T05:16:54,078 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 660158e951014a25923d9f7bc71dbdc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733116612774 2024-12-02T05:16:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/6e2c0deb9b554783ab0c3f49def4c8ec is 50, key is test_row_0/A:col10/1733116613531/Put/seqid=0 2024-12-02T05:16:54,088 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/B of d2d081cdce71d4cc4bebd39e9655d288 into 3670379f92f34703a4c5d9fe4e9eaaf1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:54,088 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:54,088 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/B, priority=12, startTime=1733116613987; duration=0sec 2024-12-02T05:16:54,088 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:54,088 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:B 2024-12-02T05:16:54,105 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:54,106 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/e4001077aca245a7bbd10f1bd606036a is 50, key is test_row_0/C:col10/1733116612779/Put/seqid=0 2024-12-02T05:16:54,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741881_1057 (size=12301) 2024-12-02T05:16:54,109 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/6e2c0deb9b554783ab0c3f49def4c8ec 2024-12-02T05:16:54,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/03dd99c954a54f26949b8846c95ea365 is 50, key is test_row_0/B:col10/1733116613531/Put/seqid=0 2024-12-02T05:16:54,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741882_1058 (size=12679) 2024-12-02T05:16:54,139 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/e4001077aca245a7bbd10f1bd606036a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/e4001077aca245a7bbd10f1bd606036a 2024-12-02T05:16:54,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741883_1059 (size=12301) 2024-12-02T05:16:54,147 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/03dd99c954a54f26949b8846c95ea365 2024-12-02T05:16:54,148 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/C of d2d081cdce71d4cc4bebd39e9655d288 into e4001077aca245a7bbd10f1bd606036a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:54,148 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:54,148 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/C, priority=12, startTime=1733116613988; duration=0sec 2024-12-02T05:16:54,148 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:54,148 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:C 2024-12-02T05:16:54,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/4664693c01f34495b5f9c72ded866ed9 is 50, key is test_row_0/C:col10/1733116613531/Put/seqid=0 2024-12-02T05:16:54,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741884_1060 (size=12301) 2024-12-02T05:16:54,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:54,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116674198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:54,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116674307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:54,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:54,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116674517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:54,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:54,565 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/4664693c01f34495b5f9c72ded866ed9 2024-12-02T05:16:54,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/6e2c0deb9b554783ab0c3f49def4c8ec as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/6e2c0deb9b554783ab0c3f49def4c8ec 2024-12-02T05:16:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T05:16:54,580 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/6e2c0deb9b554783ab0c3f49def4c8ec, entries=150, sequenceid=280, filesize=12.0 K 2024-12-02T05:16:54,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/03dd99c954a54f26949b8846c95ea365 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/03dd99c954a54f26949b8846c95ea365 2024-12-02T05:16:54,595 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/03dd99c954a54f26949b8846c95ea365, entries=150, sequenceid=280, filesize=12.0 K 2024-12-02T05:16:54,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/4664693c01f34495b5f9c72ded866ed9 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/4664693c01f34495b5f9c72ded866ed9 2024-12-02T05:16:54,607 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/4664693c01f34495b5f9c72ded866ed9, entries=150, sequenceid=280, filesize=12.0 K 2024-12-02T05:16:54,610 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d2d081cdce71d4cc4bebd39e9655d288 in 534ms, sequenceid=280, compaction requested=false 2024-12-02T05:16:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:54,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-02T05:16:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-02T05:16:54,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-02T05:16:54,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1670 sec 2024-12-02T05:16:54,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.1800 sec 2024-12-02T05:16:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:54,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T05:16:54,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:54,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:54,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:54,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:54,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:54,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:54,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/f38b9be2511644698b806c259ac4e0e3 is 50, key is test_row_0/A:col10/1733116614180/Put/seqid=0 2024-12-02T05:16:54,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741885_1061 (size=12301) 2024-12-02T05:16:54,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116674905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:54,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116674918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:54,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116674918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:54,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:54,921 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in d2d081cdce71d4cc4bebd39e9655d288 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:16:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116674918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:54,922 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in d2d081cdce71d4cc4bebd39e9655d288 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:16:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:54,923 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in d2d081cdce71d4cc4bebd39e9655d288 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:16:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:54,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116674927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:54,934 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in d2d081cdce71d4cc4bebd39e9655d288 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:16:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:55,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116675017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:55,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116675228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:55,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:55,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:55,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:55,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/f38b9be2511644698b806c259ac4e0e3 2024-12-02T05:16:55,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/98da2875b78c4638afb6fbadf4d14984 is 50, key is test_row_0/B:col10/1733116614180/Put/seqid=0 2024-12-02T05:16:55,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741886_1062 (size=12301) 2024-12-02T05:16:55,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/98da2875b78c4638afb6fbadf4d14984 2024-12-02T05:16:55,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/9f2cd475af274fbfbcd30502c9a82038 is 50, key is test_row_0/C:col10/1733116614180/Put/seqid=0 2024-12-02T05:16:55,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741887_1063 (size=12301) 2024-12-02T05:16:55,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/9f2cd475af274fbfbcd30502c9a82038 2024-12-02T05:16:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/f38b9be2511644698b806c259ac4e0e3 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f38b9be2511644698b806c259ac4e0e3 2024-12-02T05:16:55,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f38b9be2511644698b806c259ac4e0e3, entries=150, sequenceid=295, filesize=12.0 K 2024-12-02T05:16:55,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/98da2875b78c4638afb6fbadf4d14984 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/98da2875b78c4638afb6fbadf4d14984 2024-12-02T05:16:55,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/98da2875b78c4638afb6fbadf4d14984, entries=150, sequenceid=295, filesize=12.0 K 2024-12-02T05:16:55,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/9f2cd475af274fbfbcd30502c9a82038 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9f2cd475af274fbfbcd30502c9a82038 2024-12-02T05:16:55,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9f2cd475af274fbfbcd30502c9a82038, entries=150, sequenceid=295, filesize=12.0 K 2024-12-02T05:16:55,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d2d081cdce71d4cc4bebd39e9655d288 in 632ms, sequenceid=295, compaction requested=true 2024-12-02T05:16:55,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:55,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:16:55,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:55,463 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:16:55,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:16:55,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:16:55,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:16:55,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T05:16:55,463 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:16:55,465 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37281 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:16:55,466 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37281 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:16:55,466 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/B is initiating minor compaction (all files) 2024-12-02T05:16:55,466 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/A is initiating minor compaction (all files) 2024-12-02T05:16:55,466 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/B in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:55,466 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/A in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:55,466 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/6b65257b4c1b45f0a586bc52a06ec8b1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/6e2c0deb9b554783ab0c3f49def4c8ec, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f38b9be2511644698b806c259ac4e0e3] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=36.4 K 2024-12-02T05:16:55,466 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/3670379f92f34703a4c5d9fe4e9eaaf1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/03dd99c954a54f26949b8846c95ea365, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/98da2875b78c4638afb6fbadf4d14984] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=36.4 K 2024-12-02T05:16:55,466 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b65257b4c1b45f0a586bc52a06ec8b1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733116612774 2024-12-02T05:16:55,466 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3670379f92f34703a4c5d9fe4e9eaaf1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733116612774 2024-12-02T05:16:55,467 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e2c0deb9b554783ab0c3f49def4c8ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733116613499 2024-12-02T05:16:55,467 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03dd99c954a54f26949b8846c95ea365, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733116613499 2024-12-02T05:16:55,467 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting f38b9be2511644698b806c259ac4e0e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733116614180 2024-12-02T05:16:55,467 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98da2875b78c4638afb6fbadf4d14984, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733116614180 2024-12-02T05:16:55,478 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#A#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:55,479 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/d352e1c6250c48199eb6d93075a6eb4a is 50, key is test_row_0/A:col10/1733116614180/Put/seqid=0 2024-12-02T05:16:55,483 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#B#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:55,483 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/29cbcefdf876497f918e020f1fff82d1 is 50, key is test_row_0/B:col10/1733116614180/Put/seqid=0 2024-12-02T05:16:55,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741888_1064 (size=12881) 2024-12-02T05:16:55,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741889_1065 (size=12881) 2024-12-02T05:16:55,498 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/29cbcefdf876497f918e020f1fff82d1 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/29cbcefdf876497f918e020f1fff82d1 2024-12-02T05:16:55,507 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/B of d2d081cdce71d4cc4bebd39e9655d288 into 29cbcefdf876497f918e020f1fff82d1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:55,508 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:55,508 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/B, priority=13, startTime=1733116615463; duration=0sec 2024-12-02T05:16:55,508 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:16:55,508 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:B 2024-12-02T05:16:55,508 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:16:55,510 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37281 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:16:55,510 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/C is initiating minor compaction (all files) 2024-12-02T05:16:55,510 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/C in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:55,511 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/e4001077aca245a7bbd10f1bd606036a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/4664693c01f34495b5f9c72ded866ed9, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9f2cd475af274fbfbcd30502c9a82038] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=36.4 K 2024-12-02T05:16:55,511 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4001077aca245a7bbd10f1bd606036a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733116612774 2024-12-02T05:16:55,512 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4664693c01f34495b5f9c72ded866ed9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733116613499 2024-12-02T05:16:55,512 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f2cd475af274fbfbcd30502c9a82038, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733116614180 2024-12-02T05:16:55,530 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#C#compaction#53 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:55,531 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/9aff99c158994dc8983ec7df8e1423b0 is 50, key is test_row_0/C:col10/1733116614180/Put/seqid=0 2024-12-02T05:16:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:55,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T05:16:55,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:55,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:55,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:55,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:55,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:55,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:55,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/13dc21b52e8742aca44f8b917bff1b32 is 50, key is test_row_0/A:col10/1733116615538/Put/seqid=0 2024-12-02T05:16:55,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741890_1066 (size=12881) 2024-12-02T05:16:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T05:16:55,588 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:16:55,589 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:16:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-02T05:16:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T05:16:55,596 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:16:55,600 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:16:55,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:16:55,600 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/9aff99c158994dc8983ec7df8e1423b0 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9aff99c158994dc8983ec7df8e1423b0 2024-12-02T05:16:55,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116675605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:55,613 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/C of d2d081cdce71d4cc4bebd39e9655d288 into 9aff99c158994dc8983ec7df8e1423b0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:55,613 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:55,613 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/C, priority=13, startTime=1733116615463; duration=0sec 2024-12-02T05:16:55,613 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:55,613 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:C 2024-12-02T05:16:55,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741891_1067 (size=14741) 2024-12-02T05:16:55,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/13dc21b52e8742aca44f8b917bff1b32 2024-12-02T05:16:55,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/e2effde792d745f7805de88b80780c03 is 50, key is test_row_0/B:col10/1733116615538/Put/seqid=0 2024-12-02T05:16:55,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741892_1068 (size=12301) 2024-12-02T05:16:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T05:16:55,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116675717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:55,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:55,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:55,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:55,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T05:16:55,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:55,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:55,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:55,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:55,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:55,902 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/d352e1c6250c48199eb6d93075a6eb4a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/d352e1c6250c48199eb6d93075a6eb4a 2024-12-02T05:16:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T05:16:55,914 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/A of d2d081cdce71d4cc4bebd39e9655d288 into d352e1c6250c48199eb6d93075a6eb4a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:55,914 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:55,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T05:16:55,914 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/A, priority=13, startTime=1733116615463; duration=0sec 2024-12-02T05:16:55,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:55,915 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:55,915 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:A 2024-12-02T05:16:55,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:55,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:55,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:55,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:55,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116675928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:56,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/e2effde792d745f7805de88b80780c03 2024-12-02T05:16:56,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/b30f3baf9ec6420cb7c6d72bfa4b5b8d is 50, key is test_row_0/C:col10/1733116615538/Put/seqid=0 2024-12-02T05:16:56,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T05:16:56,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:56,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:56,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:56,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:56,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:56,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741893_1069 (size=12301) 2024-12-02T05:16:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T05:16:56,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T05:16:56,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:56,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:56,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:56,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:56,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:56,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116676238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:56,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:56,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:56,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:56,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T05:16:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:56,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:56,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T05:16:56,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:56,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:56,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:56,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:56,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:56,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/b30f3baf9ec6420cb7c6d72bfa4b5b8d 2024-12-02T05:16:56,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/13dc21b52e8742aca44f8b917bff1b32 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/13dc21b52e8742aca44f8b917bff1b32 2024-12-02T05:16:56,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/13dc21b52e8742aca44f8b917bff1b32, entries=200, sequenceid=316, filesize=14.4 K 2024-12-02T05:16:56,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/e2effde792d745f7805de88b80780c03 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/e2effde792d745f7805de88b80780c03 2024-12-02T05:16:56,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/e2effde792d745f7805de88b80780c03, entries=150, sequenceid=316, filesize=12.0 K 2024-12-02T05:16:56,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/b30f3baf9ec6420cb7c6d72bfa4b5b8d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/b30f3baf9ec6420cb7c6d72bfa4b5b8d 2024-12-02T05:16:56,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/b30f3baf9ec6420cb7c6d72bfa4b5b8d, entries=150, sequenceid=316, filesize=12.0 K 2024-12-02T05:16:56,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d2d081cdce71d4cc4bebd39e9655d288 in 1091ms, sequenceid=316, compaction requested=false 2024-12-02T05:16:56,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:56,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T05:16:56,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:56,703 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T05:16:56,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:56,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:56,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:56,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:56,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:56,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:56,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/1acb1fc077914b959e93ff39167b6575 is 50, key is test_row_0/A:col10/1733116615584/Put/seqid=0 2024-12-02T05:16:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T05:16:56,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741894_1070 (size=9857) 2024-12-02T05:16:56,734 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/1acb1fc077914b959e93ff39167b6575 2024-12-02T05:16:56,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/8fa0189a502d460c91b3f7f8bc7efa95 is 50, key is test_row_0/B:col10/1733116615584/Put/seqid=0 2024-12-02T05:16:56,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:56,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741895_1071 (size=9857) 2024-12-02T05:16:56,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116676839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:56,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116676947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:56,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:56,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:56,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:57,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116677157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:57,171 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/8fa0189a502d460c91b3f7f8bc7efa95 2024-12-02T05:16:57,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/978623442ed64ab09c055684441df19d is 50, key is test_row_0/C:col10/1733116615584/Put/seqid=0 2024-12-02T05:16:57,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741896_1072 (size=9857) 2024-12-02T05:16:57,188 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/978623442ed64ab09c055684441df19d 2024-12-02T05:16:57,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/1acb1fc077914b959e93ff39167b6575 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1acb1fc077914b959e93ff39167b6575 2024-12-02T05:16:57,201 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1acb1fc077914b959e93ff39167b6575, entries=100, sequenceid=330, filesize=9.6 K 2024-12-02T05:16:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/8fa0189a502d460c91b3f7f8bc7efa95 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/8fa0189a502d460c91b3f7f8bc7efa95 2024-12-02T05:16:57,209 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/8fa0189a502d460c91b3f7f8bc7efa95, entries=100, sequenceid=330, filesize=9.6 K 2024-12-02T05:16:57,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/978623442ed64ab09c055684441df19d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/978623442ed64ab09c055684441df19d 2024-12-02T05:16:57,219 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/978623442ed64ab09c055684441df19d, entries=100, sequenceid=330, filesize=9.6 K 2024-12-02T05:16:57,222 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d2d081cdce71d4cc4bebd39e9655d288 in 517ms, sequenceid=330, compaction requested=true 2024-12-02T05:16:57,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:57,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:57,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-02T05:16:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-02T05:16:57,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-02T05:16:57,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6240 sec 2024-12-02T05:16:57,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.6370 sec 2024-12-02T05:16:57,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T05:16:57,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:57,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:57,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:57,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:57,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:57,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:57,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/ad1608d2586a4bcab88ed84736b27aa3 is 50, key is test_row_0/A:col10/1733116616836/Put/seqid=0 2024-12-02T05:16:57,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741897_1073 (size=14741) 2024-12-02T05:16:57,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/ad1608d2586a4bcab88ed84736b27aa3 2024-12-02T05:16:57,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/0d5d40a2a0bf46629e40d9e28fd698dc is 50, key is test_row_0/B:col10/1733116616836/Put/seqid=0 2024-12-02T05:16:57,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741898_1074 (size=12301) 2024-12-02T05:16:57,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/0d5d40a2a0bf46629e40d9e28fd698dc 2024-12-02T05:16:57,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116677516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:57,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/54e283e37f2f417280e02f90de78857f is 50, key is test_row_0/C:col10/1733116616836/Put/seqid=0 2024-12-02T05:16:57,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741899_1075 (size=12301) 2024-12-02T05:16:57,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/54e283e37f2f417280e02f90de78857f 2024-12-02T05:16:57,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/ad1608d2586a4bcab88ed84736b27aa3 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/ad1608d2586a4bcab88ed84736b27aa3 2024-12-02T05:16:57,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/ad1608d2586a4bcab88ed84736b27aa3, entries=200, sequenceid=351, filesize=14.4 K 2024-12-02T05:16:57,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/0d5d40a2a0bf46629e40d9e28fd698dc as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0d5d40a2a0bf46629e40d9e28fd698dc 2024-12-02T05:16:57,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0d5d40a2a0bf46629e40d9e28fd698dc, entries=150, sequenceid=351, filesize=12.0 K 2024-12-02T05:16:57,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/54e283e37f2f417280e02f90de78857f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/54e283e37f2f417280e02f90de78857f 2024-12-02T05:16:57,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/54e283e37f2f417280e02f90de78857f, entries=150, sequenceid=351, filesize=12.0 K 2024-12-02T05:16:57,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for d2d081cdce71d4cc4bebd39e9655d288 in 92ms, sequenceid=351, compaction requested=true 2024-12-02T05:16:57,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:57,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:16:57,561 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:16:57,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:57,561 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:16:57,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:16:57,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:57,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:16:57,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:16:57,562 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:16:57,563 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52220 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:16:57,563 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/B is initiating minor compaction (all files) 2024-12-02T05:16:57,563 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/A is initiating minor compaction (all files) 2024-12-02T05:16:57,563 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/B in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:57,563 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/A in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:57,563 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/d352e1c6250c48199eb6d93075a6eb4a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/13dc21b52e8742aca44f8b917bff1b32, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1acb1fc077914b959e93ff39167b6575, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/ad1608d2586a4bcab88ed84736b27aa3] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=51.0 K 2024-12-02T05:16:57,563 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/29cbcefdf876497f918e020f1fff82d1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/e2effde792d745f7805de88b80780c03, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/8fa0189a502d460c91b3f7f8bc7efa95, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0d5d40a2a0bf46629e40d9e28fd698dc] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=46.2 K 2024-12-02T05:16:57,563 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 29cbcefdf876497f918e020f1fff82d1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733116614180 2024-12-02T05:16:57,563 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting d352e1c6250c48199eb6d93075a6eb4a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733116614180 2024-12-02T05:16:57,564 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting e2effde792d745f7805de88b80780c03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733116614889 2024-12-02T05:16:57,564 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13dc21b52e8742aca44f8b917bff1b32, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733116614881 2024-12-02T05:16:57,564 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fa0189a502d460c91b3f7f8bc7efa95, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733116615584 2024-12-02T05:16:57,564 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1acb1fc077914b959e93ff39167b6575, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733116615584 2024-12-02T05:16:57,565 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d5d40a2a0bf46629e40d9e28fd698dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733116616830 2024-12-02T05:16:57,565 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad1608d2586a4bcab88ed84736b27aa3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733116616822 2024-12-02T05:16:57,578 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:57,578 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#A#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:57,579 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/88d9006e0d254a0f9591040fe5ac86a3 is 50, key is test_row_0/B:col10/1733116616836/Put/seqid=0 2024-12-02T05:16:57,579 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/a64eaf41bc814da291ffe2ebbe2f8238 is 50, key is test_row_0/A:col10/1733116616836/Put/seqid=0 2024-12-02T05:16:57,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741900_1076 (size=13017) 2024-12-02T05:16:57,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741901_1077 (size=13017) 2024-12-02T05:16:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:57,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T05:16:57,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:57,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:57,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:57,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:57,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:57,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:57,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/38cfd27e776742ce9c882ad7ae2c5051 is 50, key is test_row_0/A:col10/1733116617629/Put/seqid=0 2024-12-02T05:16:57,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741902_1078 (size=12301) 2024-12-02T05:16:57,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/38cfd27e776742ce9c882ad7ae2c5051 2024-12-02T05:16:57,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/2ecf65c2b998453b83cfe208a390202d is 50, key is test_row_0/B:col10/1733116617629/Put/seqid=0 2024-12-02T05:16:57,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741903_1079 (size=12301) 2024-12-02T05:16:57,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/2ecf65c2b998453b83cfe208a390202d 2024-12-02T05:16:57,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/874cc2ff6ca64416b1657b9214cc7ea1 is 50, key is test_row_0/C:col10/1733116617629/Put/seqid=0 2024-12-02T05:16:57,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741904_1080 (size=12301) 2024-12-02T05:16:57,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/874cc2ff6ca64416b1657b9214cc7ea1 2024-12-02T05:16:57,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/38cfd27e776742ce9c882ad7ae2c5051 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/38cfd27e776742ce9c882ad7ae2c5051 2024-12-02T05:16:57,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116677726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:57,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:57,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:57,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T05:16:57,737 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:16:57,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:16:57,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/38cfd27e776742ce9c882ad7ae2c5051, entries=150, sequenceid=363, filesize=12.0 K 2024-12-02T05:16:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-02T05:16:57,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/2ecf65c2b998453b83cfe208a390202d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/2ecf65c2b998453b83cfe208a390202d 2024-12-02T05:16:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T05:16:57,741 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:16:57,742 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:16:57,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:16:57,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/2ecf65c2b998453b83cfe208a390202d, entries=150, sequenceid=363, filesize=12.0 K 2024-12-02T05:16:57,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/874cc2ff6ca64416b1657b9214cc7ea1 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/874cc2ff6ca64416b1657b9214cc7ea1 2024-12-02T05:16:57,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/874cc2ff6ca64416b1657b9214cc7ea1, entries=150, sequenceid=363, filesize=12.0 K 2024-12-02T05:16:57,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d2d081cdce71d4cc4bebd39e9655d288 in 137ms, sequenceid=363, compaction requested=true 2024-12-02T05:16:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:A, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:16:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T05:16:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:16:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T05:16:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:16:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T05:16:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:57,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T05:16:57,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:57,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:57,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:57,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:57,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:57,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:16:57,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T05:16:57,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/bb8eb7b4936b4fe4ae6215d9a7661434 is 50, key is test_row_0/A:col10/1733116617718/Put/seqid=0 2024-12-02T05:16:57,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741905_1081 (size=14741) 2024-12-02T05:16:57,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/bb8eb7b4936b4fe4ae6215d9a7661434 2024-12-02T05:16:57,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/3eb29575e11549ed9fd347b49b0db89e is 50, key is test_row_0/B:col10/1733116617718/Put/seqid=0 2024-12-02T05:16:57,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741906_1082 (size=12301) 2024-12-02T05:16:57,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/3eb29575e11549ed9fd347b49b0db89e 2024-12-02T05:16:57,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116677877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:57,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:57,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:57,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:57,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/5312e119ddaf48a1bba26a4fb86d9b51 is 50, key is test_row_0/C:col10/1733116617718/Put/seqid=0 2024-12-02T05:16:57,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T05:16:57,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:57,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:57,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:57,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:57,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:57,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741907_1083 (size=12301) 2024-12-02T05:16:57,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/5312e119ddaf48a1bba26a4fb86d9b51 2024-12-02T05:16:57,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/bb8eb7b4936b4fe4ae6215d9a7661434 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/bb8eb7b4936b4fe4ae6215d9a7661434 2024-12-02T05:16:57,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/bb8eb7b4936b4fe4ae6215d9a7661434, entries=200, sequenceid=384, filesize=14.4 K 2024-12-02T05:16:57,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/3eb29575e11549ed9fd347b49b0db89e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/3eb29575e11549ed9fd347b49b0db89e 2024-12-02T05:16:57,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/3eb29575e11549ed9fd347b49b0db89e, entries=150, sequenceid=384, filesize=12.0 K 2024-12-02T05:16:57,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/5312e119ddaf48a1bba26a4fb86d9b51 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5312e119ddaf48a1bba26a4fb86d9b51 2024-12-02T05:16:57,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5312e119ddaf48a1bba26a4fb86d9b51, entries=150, sequenceid=384, filesize=12.0 K 2024-12-02T05:16:57,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for d2d081cdce71d4cc4bebd39e9655d288 in 105ms, sequenceid=384, compaction requested=true 2024-12-02T05:16:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:A, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:16:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-12-02T05:16:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:16:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=6), splitQueue=0 2024-12-02T05:16:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:16:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2024-12-02T05:16:57,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:57,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T05:16:57,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:57,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:57,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:57,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:57,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:57,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:58,002 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/88d9006e0d254a0f9591040fe5ac86a3 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/88d9006e0d254a0f9591040fe5ac86a3 2024-12-02T05:16:58,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/1c7971580a694cb18b7c0d996a68b2c5 is 50, key is test_row_1/A:col10/1733116617988/Put/seqid=0 2024-12-02T05:16:58,012 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/a64eaf41bc814da291ffe2ebbe2f8238 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/a64eaf41bc814da291ffe2ebbe2f8238 2024-12-02T05:16:58,016 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/B of d2d081cdce71d4cc4bebd39e9655d288 into 88d9006e0d254a0f9591040fe5ac86a3(size=12.7 K), total size for store is 36.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:58,018 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:58,018 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/B, priority=12, startTime=1733116617561; duration=0sec 2024-12-02T05:16:58,018 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2024-12-02T05:16:58,019 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:B 2024-12-02T05:16:58,019 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 4 compacting, 2 eligible, 16 blocking 2024-12-02T05:16:58,022 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T05:16:58,022 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T05:16:58,023 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. because compaction request was cancelled 2024-12-02T05:16:58,023 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:A 2024-12-02T05:16:58,023 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-02T05:16:58,025 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/A of d2d081cdce71d4cc4bebd39e9655d288 into a64eaf41bc814da291ffe2ebbe2f8238(size=12.7 K), total size for store is 39.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:58,025 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 71942 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-02T05:16:58,025 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:58,026 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/A, priority=12, startTime=1733116617561; duration=0sec 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/C is initiating minor compaction (all files) 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-12-02T05:16:58,026 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/C in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:A 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:B 2024-12-02T05:16:58,026 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9aff99c158994dc8983ec7df8e1423b0, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/b30f3baf9ec6420cb7c6d72bfa4b5b8d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/978623442ed64ab09c055684441df19d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/54e283e37f2f417280e02f90de78857f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/874cc2ff6ca64416b1657b9214cc7ea1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5312e119ddaf48a1bba26a4fb86d9b51] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=70.3 K 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 6 compacting, 0 eligible, 16 blocking 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. because compaction request was cancelled 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:C 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:A 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:B 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 6 compacting, 0 eligible, 16 blocking 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aff99c158994dc8983ec7df8e1423b0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733116614180 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T05:16:58,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. because compaction request was cancelled 2024-12-02T05:16:58,027 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:C 2024-12-02T05:16:58,027 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:16:58,027 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting b30f3baf9ec6420cb7c6d72bfa4b5b8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733116614889 2024-12-02T05:16:58,027 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 978623442ed64ab09c055684441df19d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733116615584 2024-12-02T05:16:58,028 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 54e283e37f2f417280e02f90de78857f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733116616830 2024-12-02T05:16:58,028 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:16:58,028 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 874cc2ff6ca64416b1657b9214cc7ea1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733116617497 2024-12-02T05:16:58,028 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/B is initiating minor compaction (all files) 2024-12-02T05:16:58,028 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/B in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:58,029 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/88d9006e0d254a0f9591040fe5ac86a3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/2ecf65c2b998453b83cfe208a390202d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/3eb29575e11549ed9fd347b49b0db89e] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=36.7 K 2024-12-02T05:16:58,029 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88d9006e0d254a0f9591040fe5ac86a3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733116616830 2024-12-02T05:16:58,030 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ecf65c2b998453b83cfe208a390202d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733116617497 2024-12-02T05:16:58,030 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 5312e119ddaf48a1bba26a4fb86d9b51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1733116617718 2024-12-02T05:16:58,030 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eb29575e11549ed9fd347b49b0db89e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1733116617718 2024-12-02T05:16:58,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741908_1084 (size=12297) 2024-12-02T05:16:58,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/1c7971580a694cb18b7c0d996a68b2c5 2024-12-02T05:16:58,049 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#B#compaction#72 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:58,050 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/0c54574779b149958d10d8fd6786cf85 is 50, key is test_row_0/B:col10/1733116617718/Put/seqid=0 2024-12-02T05:16:58,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T05:16:58,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:58,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:58,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:58,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:58,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:58,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:58,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T05:16:58,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/67e87ea1ecae401c8a3893e34629ed59 is 50, key is test_row_1/B:col10/1733116617988/Put/seqid=0 2024-12-02T05:16:58,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741909_1085 (size=13119) 2024-12-02T05:16:58,075 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#C#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:58,076 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/e9a2773b278648d0b67adb2ef54bbb89 is 50, key is test_row_0/C:col10/1733116617718/Put/seqid=0 2024-12-02T05:16:58,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116678089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:58,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741910_1086 (size=9857) 2024-12-02T05:16:58,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/67e87ea1ecae401c8a3893e34629ed59 2024-12-02T05:16:58,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741911_1087 (size=13085) 2024-12-02T05:16:58,119 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/e9a2773b278648d0b67adb2ef54bbb89 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/e9a2773b278648d0b67adb2ef54bbb89 2024-12-02T05:16:58,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/b27ec09898ba4b7eb30d7e4b07298236 is 50, key is test_row_1/C:col10/1733116617988/Put/seqid=0 2024-12-02T05:16:58,131 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/C of d2d081cdce71d4cc4bebd39e9655d288 into e9a2773b278648d0b67adb2ef54bbb89(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:58,131 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:58,131 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/C, priority=10, startTime=1733116617946; duration=0sec 2024-12-02T05:16:58,131 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T05:16:58,132 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:C 2024-12-02T05:16:58,132 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 3 compacting, 0 eligible, 16 blocking 2024-12-02T05:16:58,132 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T05:16:58,132 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T05:16:58,132 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. because compaction request was cancelled 2024-12-02T05:16:58,132 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:B 2024-12-02T05:16:58,132 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:16:58,135 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:16:58,136 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/A is initiating minor compaction (all files) 2024-12-02T05:16:58,136 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/A in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:58,136 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/a64eaf41bc814da291ffe2ebbe2f8238, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/38cfd27e776742ce9c882ad7ae2c5051, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/bb8eb7b4936b4fe4ae6215d9a7661434] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=39.1 K 2024-12-02T05:16:58,137 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting a64eaf41bc814da291ffe2ebbe2f8238, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733116616830 2024-12-02T05:16:58,137 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 38cfd27e776742ce9c882ad7ae2c5051, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733116617497 2024-12-02T05:16:58,138 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting bb8eb7b4936b4fe4ae6215d9a7661434, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1733116617711 2024-12-02T05:16:58,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741912_1088 (size=9857) 2024-12-02T05:16:58,159 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#A#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:16:58,161 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/1d4d2fcff2b1423d8d15f39726b6e9fb is 50, key is test_row_0/A:col10/1733116617718/Put/seqid=0 2024-12-02T05:16:58,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741913_1089 (size=13119) 2024-12-02T05:16:58,182 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/1d4d2fcff2b1423d8d15f39726b6e9fb as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1d4d2fcff2b1423d8d15f39726b6e9fb 2024-12-02T05:16:58,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116678197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:58,203 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/A of d2d081cdce71d4cc4bebd39e9655d288 into 1d4d2fcff2b1423d8d15f39726b6e9fb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:58,203 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:58,203 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/A, priority=13, startTime=1733116617768; duration=0sec 2024-12-02T05:16:58,203 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:58,203 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:A 2024-12-02T05:16:58,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T05:16:58,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:58,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:58,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:58,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:58,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:58,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T05:16:58,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:58,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:58,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:58,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:58,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T05:16:58,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116678407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:58,476 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/0c54574779b149958d10d8fd6786cf85 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0c54574779b149958d10d8fd6786cf85 2024-12-02T05:16:58,483 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/B of d2d081cdce71d4cc4bebd39e9655d288 into 0c54574779b149958d10d8fd6786cf85(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:16:58,484 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:58,484 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/B, priority=13, startTime=1733116617768; duration=0sec 2024-12-02T05:16:58,484 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:16:58,484 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:B 2024-12-02T05:16:58,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T05:16:58,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:58,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:58,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:58,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:58,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:16:58,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/b27ec09898ba4b7eb30d7e4b07298236 2024-12-02T05:16:58,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/1c7971580a694cb18b7c0d996a68b2c5 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1c7971580a694cb18b7c0d996a68b2c5 2024-12-02T05:16:58,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1c7971580a694cb18b7c0d996a68b2c5, entries=150, sequenceid=396, filesize=12.0 K 2024-12-02T05:16:58,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/67e87ea1ecae401c8a3893e34629ed59 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/67e87ea1ecae401c8a3893e34629ed59 2024-12-02T05:16:58,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/67e87ea1ecae401c8a3893e34629ed59, entries=100, sequenceid=396, filesize=9.6 K 2024-12-02T05:16:58,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/b27ec09898ba4b7eb30d7e4b07298236 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/b27ec09898ba4b7eb30d7e4b07298236 2024-12-02T05:16:58,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/b27ec09898ba4b7eb30d7e4b07298236, entries=100, sequenceid=396, filesize=9.6 K 2024-12-02T05:16:58,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d2d081cdce71d4cc4bebd39e9655d288 in 594ms, sequenceid=396, compaction requested=false 2024-12-02T05:16:58,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:58,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T05:16:58,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:58,672 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T05:16:58,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:58,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:58,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:58,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:58,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:58,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:58,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/f80a52f383dc441ba991bd7e6ab8e5ad is 50, key is test_row_0/A:col10/1733116618061/Put/seqid=0 2024-12-02T05:16:58,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741914_1090 (size=12301) 2024-12-02T05:16:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:58,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:16:58,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116678746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:58,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:58,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116678857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T05:16:58,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35360 deadline: 1733116678947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:58,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35310 deadline: 1733116678948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:58,949 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in d2d081cdce71d4cc4bebd39e9655d288 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:16:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:58,950 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in d2d081cdce71d4cc4bebd39e9655d288 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:16:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:58,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35312 deadline: 1733116678957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:58,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:58,959 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in d2d081cdce71d4cc4bebd39e9655d288 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:16:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35346 deadline: 1733116678957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:58,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:58,961 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in d2d081cdce71d4cc4bebd39e9655d288 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:16:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:59,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116679067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:59,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:59,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:59,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:59,084 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/f80a52f383dc441ba991bd7e6ab8e5ad 2024-12-02T05:16:59,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/381d8759ee094154a51528200584c8ed is 50, key is test_row_0/B:col10/1733116618061/Put/seqid=0 2024-12-02T05:16:59,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741915_1091 (size=12301) 2024-12-02T05:16:59,108 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/381d8759ee094154a51528200584c8ed 2024-12-02T05:16:59,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/0875d1c3ac3f4876bed6ec62c7e5352f is 50, key is test_row_0/C:col10/1733116618061/Put/seqid=0 2024-12-02T05:16:59,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741916_1092 (size=12301) 2024-12-02T05:16:59,129 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/0875d1c3ac3f4876bed6ec62c7e5352f 2024-12-02T05:16:59,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/f80a52f383dc441ba991bd7e6ab8e5ad as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f80a52f383dc441ba991bd7e6ab8e5ad 2024-12-02T05:16:59,149 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f80a52f383dc441ba991bd7e6ab8e5ad, entries=150, sequenceid=421, filesize=12.0 K 2024-12-02T05:16:59,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/381d8759ee094154a51528200584c8ed as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/381d8759ee094154a51528200584c8ed 2024-12-02T05:16:59,158 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/381d8759ee094154a51528200584c8ed, entries=150, sequenceid=421, filesize=12.0 K 2024-12-02T05:16:59,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/0875d1c3ac3f4876bed6ec62c7e5352f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/0875d1c3ac3f4876bed6ec62c7e5352f 2024-12-02T05:16:59,165 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/0875d1c3ac3f4876bed6ec62c7e5352f, entries=150, sequenceid=421, filesize=12.0 K 2024-12-02T05:16:59,166 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d2d081cdce71d4cc4bebd39e9655d288 in 494ms, sequenceid=421, compaction requested=true 2024-12-02T05:16:59,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:16:59,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:16:59,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-02T05:16:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-02T05:16:59,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-02T05:16:59,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-12-02T05:16:59,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.4340 sec 2024-12-02T05:16:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:16:59,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T05:16:59,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:16:59,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:59,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:16:59,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:59,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:16:59,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:16:59,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/15e8ca9cb39a4df39429f04be9841d7b is 50, key is test_row_1/A:col10/1733116618744/Put/seqid=0 2024-12-02T05:16:59,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741917_1093 (size=9857) 2024-12-02T05:16:59,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/15e8ca9cb39a4df39429f04be9841d7b 2024-12-02T05:16:59,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/38f359e226a44fba8a6125d4f8756924 is 50, key is test_row_1/B:col10/1733116618744/Put/seqid=0 2024-12-02T05:16:59,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741918_1094 (size=9857) 2024-12-02T05:16:59,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116679445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:59,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:59,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:59,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:59,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116679557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:59,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:59,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:59,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:59,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:16:59,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116679767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:16:59,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:59,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:16:59,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:16:59,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/38f359e226a44fba8a6125d4f8756924 2024-12-02T05:16:59,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/d689fa0062dc4f7ea99586de6ba2c06d is 50, key is test_row_1/C:col10/1733116618744/Put/seqid=0 2024-12-02T05:16:59,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741919_1095 (size=9857) 2024-12-02T05:16:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T05:16:59,888 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:16:59,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:16:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-02T05:16:59,891 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:16:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T05:16:59,892 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:16:59,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:16:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T05:17:00,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T05:17:00,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:00,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:17:00,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:00,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:00,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:00,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116680078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:00,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:00,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:00,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:00,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T05:17:00,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:00,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:17:00,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:00,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:00,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T05:17:00,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/d689fa0062dc4f7ea99586de6ba2c06d 2024-12-02T05:17:00,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/15e8ca9cb39a4df39429f04be9841d7b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/15e8ca9cb39a4df39429f04be9841d7b 2024-12-02T05:17:00,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/15e8ca9cb39a4df39429f04be9841d7b, entries=100, sequenceid=434, filesize=9.6 K 2024-12-02T05:17:00,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/38f359e226a44fba8a6125d4f8756924 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/38f359e226a44fba8a6125d4f8756924 2024-12-02T05:17:00,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/38f359e226a44fba8a6125d4f8756924, entries=100, sequenceid=434, filesize=9.6 K 2024-12-02T05:17:00,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/d689fa0062dc4f7ea99586de6ba2c06d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d689fa0062dc4f7ea99586de6ba2c06d 2024-12-02T05:17:00,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d689fa0062dc4f7ea99586de6ba2c06d, entries=100, sequenceid=434, filesize=9.6 K 2024-12-02T05:17:00,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d2d081cdce71d4cc4bebd39e9655d288 in 898ms, sequenceid=434, compaction requested=true 2024-12-02T05:17:00,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:17:00,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:17:00,278 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:17:00,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:00,278 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:17:00,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:17:00,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:00,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:17:00,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:00,280 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47574 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:17:00,280 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45134 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:17:00,280 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/A is initiating minor compaction (all files) 2024-12-02T05:17:00,280 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/B is initiating minor compaction (all files) 2024-12-02T05:17:00,280 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/A in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:00,280 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/B in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:00,280 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1d4d2fcff2b1423d8d15f39726b6e9fb, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1c7971580a694cb18b7c0d996a68b2c5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f80a52f383dc441ba991bd7e6ab8e5ad, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/15e8ca9cb39a4df39429f04be9841d7b] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=46.5 K 2024-12-02T05:17:00,280 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0c54574779b149958d10d8fd6786cf85, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/67e87ea1ecae401c8a3893e34629ed59, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/381d8759ee094154a51528200584c8ed, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/38f359e226a44fba8a6125d4f8756924] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=44.1 K 2024-12-02T05:17:00,281 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d4d2fcff2b1423d8d15f39726b6e9fb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1733116617718 2024-12-02T05:17:00,281 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c54574779b149958d10d8fd6786cf85, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1733116617718 2024-12-02T05:17:00,281 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 67e87ea1ecae401c8a3893e34629ed59, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1733116617855 2024-12-02T05:17:00,281 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c7971580a694cb18b7c0d996a68b2c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1733116617855 2024-12-02T05:17:00,281 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 381d8759ee094154a51528200584c8ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733116618061 2024-12-02T05:17:00,281 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting f80a52f383dc441ba991bd7e6ab8e5ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733116618061 2024-12-02T05:17:00,282 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 38f359e226a44fba8a6125d4f8756924, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1733116618744 2024-12-02T05:17:00,282 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15e8ca9cb39a4df39429f04be9841d7b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1733116618744 2024-12-02T05:17:00,297 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#B#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:00,298 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/dd259089a25e48ac8d1fa48a4a86f199 is 50, key is test_row_0/B:col10/1733116618061/Put/seqid=0 2024-12-02T05:17:00,310 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:00,311 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/2a5735c70efb4af4b5fc174594abdd19 is 50, key is test_row_0/A:col10/1733116618061/Put/seqid=0 2024-12-02T05:17:00,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741920_1096 (size=13255) 2024-12-02T05:17:00,352 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/dd259089a25e48ac8d1fa48a4a86f199 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/dd259089a25e48ac8d1fa48a4a86f199 2024-12-02T05:17:00,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T05:17:00,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:00,354 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T05:17:00,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:17:00,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:00,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:17:00,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:00,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:17:00,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:00,368 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/B of d2d081cdce71d4cc4bebd39e9655d288 into dd259089a25e48ac8d1fa48a4a86f199(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:00,368 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:17:00,368 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/B, priority=12, startTime=1733116620278; duration=0sec 2024-12-02T05:17:00,368 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:00,368 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:B 2024-12-02T05:17:00,368 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:17:00,373 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45100 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:17:00,373 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/C is initiating minor compaction (all files) 2024-12-02T05:17:00,373 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/C in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:00,373 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/e9a2773b278648d0b67adb2ef54bbb89, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/b27ec09898ba4b7eb30d7e4b07298236, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/0875d1c3ac3f4876bed6ec62c7e5352f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d689fa0062dc4f7ea99586de6ba2c06d] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=44.0 K 2024-12-02T05:17:00,374 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting e9a2773b278648d0b67adb2ef54bbb89, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1733116617718 2024-12-02T05:17:00,375 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting b27ec09898ba4b7eb30d7e4b07298236, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1733116617855 2024-12-02T05:17:00,376 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 0875d1c3ac3f4876bed6ec62c7e5352f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733116618061 2024-12-02T05:17:00,376 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting d689fa0062dc4f7ea99586de6ba2c06d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1733116618744 2024-12-02T05:17:00,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741921_1097 (size=13255) 2024-12-02T05:17:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/12aefb2dbb5640598107618a7c6e0fcf is 50, key is test_row_0/A:col10/1733116619433/Put/seqid=0 2024-12-02T05:17:00,391 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/2a5735c70efb4af4b5fc174594abdd19 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/2a5735c70efb4af4b5fc174594abdd19 2024-12-02T05:17:00,401 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/A of d2d081cdce71d4cc4bebd39e9655d288 into 2a5735c70efb4af4b5fc174594abdd19(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:00,401 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:17:00,401 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/A, priority=12, startTime=1733116620278; duration=0sec 2024-12-02T05:17:00,401 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:00,401 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:A 2024-12-02T05:17:00,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741922_1098 (size=12301) 2024-12-02T05:17:00,411 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/12aefb2dbb5640598107618a7c6e0fcf 2024-12-02T05:17:00,413 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#C#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:00,415 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/5464fd0ed33b4f12b7bf0dc52da63bc8 is 50, key is test_row_0/C:col10/1733116618061/Put/seqid=0 2024-12-02T05:17:00,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/12693dd73bbd4ceeb81dafa83b40c1fa is 50, key is test_row_0/B:col10/1733116619433/Put/seqid=0 2024-12-02T05:17:00,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741923_1099 (size=13221) 2024-12-02T05:17:00,451 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/5464fd0ed33b4f12b7bf0dc52da63bc8 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5464fd0ed33b4f12b7bf0dc52da63bc8 2024-12-02T05:17:00,464 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/C of d2d081cdce71d4cc4bebd39e9655d288 into 5464fd0ed33b4f12b7bf0dc52da63bc8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:00,464 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:17:00,464 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/C, priority=12, startTime=1733116620278; duration=0sec 2024-12-02T05:17:00,464 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:00,465 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:C 2024-12-02T05:17:00,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741924_1100 (size=12301) 2024-12-02T05:17:00,486 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/12693dd73bbd4ceeb81dafa83b40c1fa 2024-12-02T05:17:00,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/67fe5645c1ef4669a9842f60b04af4cf is 50, key is test_row_0/C:col10/1733116619433/Put/seqid=0 2024-12-02T05:17:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T05:17:00,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741925_1101 (size=12301) 2024-12-02T05:17:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:17:00,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:17:00,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116680622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:00,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116680727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:00,921 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/67fe5645c1ef4669a9842f60b04af4cf 2024-12-02T05:17:00,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/12aefb2dbb5640598107618a7c6e0fcf as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/12aefb2dbb5640598107618a7c6e0fcf 2024-12-02T05:17:00,935 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/12aefb2dbb5640598107618a7c6e0fcf, entries=150, sequenceid=453, filesize=12.0 K 2024-12-02T05:17:00,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/12693dd73bbd4ceeb81dafa83b40c1fa as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/12693dd73bbd4ceeb81dafa83b40c1fa 2024-12-02T05:17:00,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116680938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:00,957 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/12693dd73bbd4ceeb81dafa83b40c1fa, entries=150, sequenceid=453, filesize=12.0 K 2024-12-02T05:17:00,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/67fe5645c1ef4669a9842f60b04af4cf as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/67fe5645c1ef4669a9842f60b04af4cf 2024-12-02T05:17:00,963 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/67fe5645c1ef4669a9842f60b04af4cf, entries=150, sequenceid=453, filesize=12.0 K 2024-12-02T05:17:00,964 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d2d081cdce71d4cc4bebd39e9655d288 in 610ms, sequenceid=453, compaction requested=false 2024-12-02T05:17:00,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:17:00,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:00,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-02T05:17:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-02T05:17:00,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-02T05:17:00,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0730 sec 2024-12-02T05:17:00,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.0810 sec 2024-12-02T05:17:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T05:17:01,027 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:01,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-02T05:17:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T05:17:01,030 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:01,030 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:01,031 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T05:17:01,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T05:17:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:01,183 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T05:17:01,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:17:01,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:01,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:17:01,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:01,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:17:01,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:01,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/b6740a1aceef460db1f859381e9eb7ca is 50, key is test_row_0/A:col10/1733116620603/Put/seqid=0 2024-12-02T05:17:01,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741926_1102 (size=12301) 2024-12-02T05:17:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:17:01,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:17:01,278 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T05:17:01,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 318 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116681321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:01,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:01,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:01,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T05:17:01,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116681428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:01,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116681637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T05:17:01,799 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/b6740a1aceef460db1f859381e9eb7ca 2024-12-02T05:17:01,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/59a8b4e63e964f23ac3dbcc3911e3017 is 50, key is test_row_0/B:col10/1733116620603/Put/seqid=0 2024-12-02T05:17:01,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741927_1103 (size=12301) 2024-12-02T05:17:01,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116681947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T05:17:02,220 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/59a8b4e63e964f23ac3dbcc3911e3017 2024-12-02T05:17:02,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/84547d042c204df9bc87eedf20ba2f47 is 50, key is test_row_0/C:col10/1733116620603/Put/seqid=0 2024-12-02T05:17:02,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741928_1104 (size=12301) 2024-12-02T05:17:02,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:02,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116682457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:02,642 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/84547d042c204df9bc87eedf20ba2f47 2024-12-02T05:17:02,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/b6740a1aceef460db1f859381e9eb7ca as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/b6740a1aceef460db1f859381e9eb7ca 2024-12-02T05:17:02,654 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/b6740a1aceef460db1f859381e9eb7ca, entries=150, sequenceid=469, filesize=12.0 K 2024-12-02T05:17:02,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/59a8b4e63e964f23ac3dbcc3911e3017 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/59a8b4e63e964f23ac3dbcc3911e3017 2024-12-02T05:17:02,660 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/59a8b4e63e964f23ac3dbcc3911e3017, entries=150, sequenceid=469, filesize=12.0 K 2024-12-02T05:17:02,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/84547d042c204df9bc87eedf20ba2f47 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/84547d042c204df9bc87eedf20ba2f47 2024-12-02T05:17:02,667 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/84547d042c204df9bc87eedf20ba2f47, entries=150, sequenceid=469, filesize=12.0 K 2024-12-02T05:17:02,668 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d2d081cdce71d4cc4bebd39e9655d288 in 1485ms, sequenceid=469, compaction requested=true 2024-12-02T05:17:02,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:17:02,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:02,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-02T05:17:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-02T05:17:02,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-02T05:17:02,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6400 sec 2024-12-02T05:17:02,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.6450 sec 2024-12-02T05:17:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T05:17:03,187 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:03,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-02T05:17:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T05:17:03,193 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:03,194 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:03,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T05:17:03,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T05:17:03,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:03,347 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T05:17:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:17:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:17:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:17:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:03,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/39d2c9ee5557461c9077cbed167b19f4 is 50, key is test_row_0/A:col10/1733116621309/Put/seqid=0 2024-12-02T05:17:03,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741929_1105 (size=12301) 2024-12-02T05:17:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:17:03,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. as already flushing 2024-12-02T05:17:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T05:17:03,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 333 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116683518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:03,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116683628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:03,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:03,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:03,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:03,766 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/39d2c9ee5557461c9077cbed167b19f4 2024-12-02T05:17:03,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/4e5e514f57f84827b74be099d6bdffcd is 50, key is test_row_0/B:col10/1733116621309/Put/seqid=0 2024-12-02T05:17:03,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741930_1106 (size=12301) 2024-12-02T05:17:03,802 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/4e5e514f57f84827b74be099d6bdffcd 2024-12-02T05:17:03,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/25f31b1300f1466d90e3f61ab0a6cb69 is 50, key is test_row_0/C:col10/1733116621309/Put/seqid=0 2024-12-02T05:17:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T05:17:03,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741931_1107 (size=12301) 2024-12-02T05:17:03,826 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/25f31b1300f1466d90e3f61ab0a6cb69 2024-12-02T05:17:03,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 335 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1733116683838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:03,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:03,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:03,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:03,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/39d2c9ee5557461c9077cbed167b19f4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/39d2c9ee5557461c9077cbed167b19f4 2024-12-02T05:17:03,868 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/39d2c9ee5557461c9077cbed167b19f4, entries=150, sequenceid=488, filesize=12.0 K 2024-12-02T05:17:03,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/4e5e514f57f84827b74be099d6bdffcd as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/4e5e514f57f84827b74be099d6bdffcd 2024-12-02T05:17:03,877 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/4e5e514f57f84827b74be099d6bdffcd, entries=150, sequenceid=488, filesize=12.0 K 2024-12-02T05:17:03,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/25f31b1300f1466d90e3f61ab0a6cb69 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/25f31b1300f1466d90e3f61ab0a6cb69 2024-12-02T05:17:03,886 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/25f31b1300f1466d90e3f61ab0a6cb69, entries=150, sequenceid=488, filesize=12.0 K 2024-12-02T05:17:03,887 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d2d081cdce71d4cc4bebd39e9655d288 in 540ms, sequenceid=488, compaction requested=true 2024-12-02T05:17:03,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:17:03,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:03,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-02T05:17:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-02T05:17:03,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-02T05:17:03,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 695 msec 2024-12-02T05:17:03,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 705 msec 2024-12-02T05:17:03,913 INFO [Thread-157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-157. 2024-12-02T05:17:03,913 DEBUG [Thread-157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:03,915 INFO [Thread-155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-155. 2024-12-02T05:17:03,915 DEBUG [Thread-155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:03,916 INFO [Thread-161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-161. 2024-12-02T05:17:03,916 DEBUG [Thread-161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:03,919 INFO [Thread-159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-159. 2024-12-02T05:17:03,919 DEBUG [Thread-159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:03,920 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:03,920 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:03,920 DEBUG [Thread-161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:03,920 DEBUG [Thread-159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:03,921 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:03,921 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:03,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:03,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:03,921 DEBUG [Thread-159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:03,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:03,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:03,923 DEBUG [Thread-161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:17:04,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T05:17:04,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:17:04,150 INFO [Thread-148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-148. 2024-12-02T05:17:04,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:04,150 DEBUG [Thread-148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:04,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:17:04,150 DEBUG [Thread-148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:04,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:04,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:17:04,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:04,151 DEBUG [Thread-148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:04,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:04,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/1f17b27fc8f546ec94a409c983891f25 is 50, key is test_row_0/A:col10/1733116623515/Put/seqid=0 2024-12-02T05:17:04,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741932_1108 (size=12301) 2024-12-02T05:17:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T05:17:04,328 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:04,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/1f17b27fc8f546ec94a409c983891f25 2024-12-02T05:17:04,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/11ecbd97c4b3416197f7637e23295d98 is 50, key is test_row_0/B:col10/1733116623515/Put/seqid=0 2024-12-02T05:17:04,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741933_1109 (size=12301) 2024-12-02T05:17:04,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/11ecbd97c4b3416197f7637e23295d98 2024-12-02T05:17:04,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/abf4b838abf54a4b8fc685b76551ab3b is 50, key is test_row_0/C:col10/1733116623515/Put/seqid=0 2024-12-02T05:17:04,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741934_1110 (size=12301) 2024-12-02T05:17:04,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/abf4b838abf54a4b8fc685b76551ab3b 2024-12-02T05:17:05,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/1f17b27fc8f546ec94a409c983891f25 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1f17b27fc8f546ec94a409c983891f25 2024-12-02T05:17:05,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1f17b27fc8f546ec94a409c983891f25, entries=150, sequenceid=502, filesize=12.0 K 2024-12-02T05:17:05,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/11ecbd97c4b3416197f7637e23295d98 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/11ecbd97c4b3416197f7637e23295d98 2024-12-02T05:17:05,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/11ecbd97c4b3416197f7637e23295d98, entries=150, sequenceid=502, filesize=12.0 K 2024-12-02T05:17:05,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/abf4b838abf54a4b8fc685b76551ab3b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/abf4b838abf54a4b8fc685b76551ab3b 2024-12-02T05:17:05,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/abf4b838abf54a4b8fc685b76551ab3b, entries=150, sequenceid=502, filesize=12.0 K 2024-12-02T05:17:05,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for d2d081cdce71d4cc4bebd39e9655d288 in 874ms, sequenceid=502, compaction requested=true 2024-12-02T05:17:05,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:17:05,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:17:05,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:05,025 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T05:17:05,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:17:05,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:05,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2d081cdce71d4cc4bebd39e9655d288:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:17:05,025 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T05:17:05,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:05,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62459 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T05:17:05,026 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62459 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T05:17:05,026 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/B is initiating minor compaction (all files) 2024-12-02T05:17:05,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/A is initiating minor compaction (all files) 2024-12-02T05:17:05,026 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/A in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:05,026 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/B in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:05,026 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/2a5735c70efb4af4b5fc174594abdd19, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/12aefb2dbb5640598107618a7c6e0fcf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/b6740a1aceef460db1f859381e9eb7ca, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/39d2c9ee5557461c9077cbed167b19f4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1f17b27fc8f546ec94a409c983891f25] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=61.0 K 2024-12-02T05:17:05,026 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/dd259089a25e48ac8d1fa48a4a86f199, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/12693dd73bbd4ceeb81dafa83b40c1fa, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/59a8b4e63e964f23ac3dbcc3911e3017, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/4e5e514f57f84827b74be099d6bdffcd, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/11ecbd97c4b3416197f7637e23295d98] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=61.0 K 2024-12-02T05:17:05,027 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a5735c70efb4af4b5fc174594abdd19, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1733116618061 2024-12-02T05:17:05,027 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting dd259089a25e48ac8d1fa48a4a86f199, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1733116618061 2024-12-02T05:17:05,027 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12aefb2dbb5640598107618a7c6e0fcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1733116619426 2024-12-02T05:17:05,027 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 12693dd73bbd4ceeb81dafa83b40c1fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1733116619426 2024-12-02T05:17:05,027 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6740a1aceef460db1f859381e9eb7ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733116620603 2024-12-02T05:17:05,027 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a8b4e63e964f23ac3dbcc3911e3017, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733116620603 2024-12-02T05:17:05,028 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39d2c9ee5557461c9077cbed167b19f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1733116621305 2024-12-02T05:17:05,028 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e5e514f57f84827b74be099d6bdffcd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1733116621305 2024-12-02T05:17:05,028 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f17b27fc8f546ec94a409c983891f25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733116623512 2024-12-02T05:17:05,028 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 11ecbd97c4b3416197f7637e23295d98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733116623512 2024-12-02T05:17:05,037 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#B#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:05,037 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:05,037 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/bdd245916c0047e09ea18f875f508fd5 is 50, key is test_row_0/A:col10/1733116623515/Put/seqid=0 2024-12-02T05:17:05,037 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/fc9645751fb845098bd1207edfc76394 is 50, key is test_row_0/B:col10/1733116623515/Put/seqid=0 2024-12-02T05:17:05,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741936_1112 (size=13425) 2024-12-02T05:17:05,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741935_1111 (size=13425) 2024-12-02T05:17:05,453 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/bdd245916c0047e09ea18f875f508fd5 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/bdd245916c0047e09ea18f875f508fd5 2024-12-02T05:17:05,453 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/fc9645751fb845098bd1207edfc76394 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/fc9645751fb845098bd1207edfc76394 2024-12-02T05:17:05,459 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/A of d2d081cdce71d4cc4bebd39e9655d288 into bdd245916c0047e09ea18f875f508fd5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:05,459 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/B of d2d081cdce71d4cc4bebd39e9655d288 into fc9645751fb845098bd1207edfc76394(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:05,460 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:17:05,460 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:17:05,460 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/A, priority=11, startTime=1733116625024; duration=0sec 2024-12-02T05:17:05,460 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/B, priority=11, startTime=1733116625025; duration=0sec 2024-12-02T05:17:05,460 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:05,460 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:A 2024-12-02T05:17:05,460 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:05,460 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:B 2024-12-02T05:17:05,460 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T05:17:05,462 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62425 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T05:17:05,462 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): d2d081cdce71d4cc4bebd39e9655d288/C is initiating minor compaction (all files) 2024-12-02T05:17:05,462 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2d081cdce71d4cc4bebd39e9655d288/C in TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:05,462 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5464fd0ed33b4f12b7bf0dc52da63bc8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/67fe5645c1ef4669a9842f60b04af4cf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/84547d042c204df9bc87eedf20ba2f47, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/25f31b1300f1466d90e3f61ab0a6cb69, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/abf4b838abf54a4b8fc685b76551ab3b] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp, totalSize=61.0 K 2024-12-02T05:17:05,462 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5464fd0ed33b4f12b7bf0dc52da63bc8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1733116618061 2024-12-02T05:17:05,462 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67fe5645c1ef4669a9842f60b04af4cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1733116619426 2024-12-02T05:17:05,463 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84547d042c204df9bc87eedf20ba2f47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733116620603 2024-12-02T05:17:05,463 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25f31b1300f1466d90e3f61ab0a6cb69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1733116621305 2024-12-02T05:17:05,463 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting abf4b838abf54a4b8fc685b76551ab3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733116623512 2024-12-02T05:17:05,473 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2d081cdce71d4cc4bebd39e9655d288#C#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:05,474 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/53deb584fb7a476bac0d220df6144494 is 50, key is test_row_0/C:col10/1733116623515/Put/seqid=0 2024-12-02T05:17:05,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741937_1113 (size=13391) 2024-12-02T05:17:05,891 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/53deb584fb7a476bac0d220df6144494 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/53deb584fb7a476bac0d220df6144494 2024-12-02T05:17:05,899 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d2d081cdce71d4cc4bebd39e9655d288/C of d2d081cdce71d4cc4bebd39e9655d288 into 53deb584fb7a476bac0d220df6144494(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:05,899 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2d081cdce71d4cc4bebd39e9655d288: 2024-12-02T05:17:05,899 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., storeName=d2d081cdce71d4cc4bebd39e9655d288/C, priority=11, startTime=1733116625025; duration=0sec 2024-12-02T05:17:05,899 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:05,899 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2d081cdce71d4cc4bebd39e9655d288:C 2024-12-02T05:17:09,001 INFO [Thread-146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-146. 2024-12-02T05:17:09,001 DEBUG [Thread-146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:09,002 DEBUG [Thread-146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:09,002 DEBUG [Thread-146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:09,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:09,020 INFO [Thread-144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-144. 2024-12-02T05:17:09,020 DEBUG [Thread-144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:09,020 DEBUG [Thread-144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:09,021 DEBUG [Thread-144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:09,021 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:09,041 INFO [Thread-150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-150. 2024-12-02T05:17:09,041 DEBUG [Thread-150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:09,041 DEBUG [Thread-150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:09,041 DEBUG [Thread-150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:09,041 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:09,050 INFO [Thread-152 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-152. 2024-12-02T05:17:09,050 DEBUG [Thread-152 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:09,050 DEBUG [Thread-152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:09,050 DEBUG [Thread-152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:09,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:09,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T05:17:09,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-12-02T05:17:09,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-12-02T05:17:09,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 265 2024-12-02T05:17:09,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-12-02T05:17:09,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-12-02T05:17:09,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T05:17:09,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3815 2024-12-02T05:17:09,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3515 2024-12-02T05:17:09,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T05:17:09,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1656 2024-12-02T05:17:09,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4963 rows 2024-12-02T05:17:09,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1669 2024-12-02T05:17:09,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5005 rows 2024-12-02T05:17:09,052 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T05:17:09,052 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T05:17:09,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:09,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:09,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:09,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T05:17:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T05:17:09,065 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116629065"}]},"ts":"1733116629065"} 2024-12-02T05:17:09,067 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T05:17:09,067 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T05:17:09,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T05:17:09,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2d081cdce71d4cc4bebd39e9655d288, UNASSIGN}] 2024-12-02T05:17:09,073 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2d081cdce71d4cc4bebd39e9655d288, UNASSIGN 2024-12-02T05:17:09,075 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=d2d081cdce71d4cc4bebd39e9655d288, regionState=CLOSING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:17:09,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2d081cdce71d4cc4bebd39e9655d288, UNASSIGN because future has completed 2024-12-02T05:17:09,077 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T05:17:09,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:17:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T05:17:09,235 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:17:09,235 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T05:17:09,235 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing d2d081cdce71d4cc4bebd39e9655d288, disabling compactions & flushes 2024-12-02T05:17:09,235 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:09,235 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:09,235 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. after waiting 0 ms 2024-12-02T05:17:09,236 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:09,236 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing d2d081cdce71d4cc4bebd39e9655d288 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-02T05:17:09,236 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=A 2024-12-02T05:17:09,236 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:09,236 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=B 2024-12-02T05:17:09,236 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:09,236 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d2d081cdce71d4cc4bebd39e9655d288, store=C 2024-12-02T05:17:09,236 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:09,240 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/dd81367304f04df486d26af40398054f is 50, key is test_row_1/A:col10/1733116629018/Put/seqid=0 2024-12-02T05:17:09,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741938_1114 (size=9857) 2024-12-02T05:17:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T05:17:09,646 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/dd81367304f04df486d26af40398054f 2024-12-02T05:17:09,660 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/4e873b98e1084d1c9f159a23bd9d1cb4 is 50, key is test_row_1/B:col10/1733116629018/Put/seqid=0 2024-12-02T05:17:09,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741939_1115 (size=9857) 2024-12-02T05:17:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T05:17:10,066 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/4e873b98e1084d1c9f159a23bd9d1cb4 2024-12-02T05:17:10,080 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/edb62086370446c4a9b92f0556bfcc35 is 50, key is test_row_1/C:col10/1733116629018/Put/seqid=0 2024-12-02T05:17:10,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741940_1116 (size=9857) 2024-12-02T05:17:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T05:17:10,488 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/edb62086370446c4a9b92f0556bfcc35 2024-12-02T05:17:10,496 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/A/dd81367304f04df486d26af40398054f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/dd81367304f04df486d26af40398054f 2024-12-02T05:17:10,503 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/dd81367304f04df486d26af40398054f, entries=100, sequenceid=512, filesize=9.6 K 2024-12-02T05:17:10,504 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/B/4e873b98e1084d1c9f159a23bd9d1cb4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/4e873b98e1084d1c9f159a23bd9d1cb4 2024-12-02T05:17:10,509 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/4e873b98e1084d1c9f159a23bd9d1cb4, entries=100, sequenceid=512, filesize=9.6 K 2024-12-02T05:17:10,511 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/.tmp/C/edb62086370446c4a9b92f0556bfcc35 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/edb62086370446c4a9b92f0556bfcc35 2024-12-02T05:17:10,517 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/edb62086370446c4a9b92f0556bfcc35, entries=100, sequenceid=512, filesize=9.6 K 2024-12-02T05:17:10,518 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d2d081cdce71d4cc4bebd39e9655d288 in 1282ms, sequenceid=512, compaction requested=false 2024-12-02T05:17:10,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/3e86bad8268a46d38947f761d74a2479, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/c2fb629c636941eea45b18bbb7cba694, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/a1677ea89e7846deae68fbfb8dd06658, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/fab0afbf8ddb4098bd2bc1eab55d4e12, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/dc26811fa1f44683bc6af87696122fa9, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/7aa9408897ab415f922e671853f6f5de, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/79ae191becae493cb17ef12b942371ff, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f1b2ed3e9a34428097a1d73ffe5ad0fa, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/8b84a59768c14bc9880ab6a697231f7b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/51a8c32b23ff4172a386f7fce00e7d53, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/c31abbbd27e94fa0ade31b7665e721f4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/7fc33ff89037455dacd0e8cac1f47934, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/084bfbdecc864749ae061f35a22ac02a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/5d95780d5f9f480d89adfa56f99d1d14, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/6b65257b4c1b45f0a586bc52a06ec8b1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/6e2c0deb9b554783ab0c3f49def4c8ec, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/d352e1c6250c48199eb6d93075a6eb4a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f38b9be2511644698b806c259ac4e0e3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/13dc21b52e8742aca44f8b917bff1b32, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1acb1fc077914b959e93ff39167b6575, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/ad1608d2586a4bcab88ed84736b27aa3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/a64eaf41bc814da291ffe2ebbe2f8238, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/38cfd27e776742ce9c882ad7ae2c5051, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/bb8eb7b4936b4fe4ae6215d9a7661434, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1d4d2fcff2b1423d8d15f39726b6e9fb, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1c7971580a694cb18b7c0d996a68b2c5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f80a52f383dc441ba991bd7e6ab8e5ad, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/2a5735c70efb4af4b5fc174594abdd19, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/15e8ca9cb39a4df39429f04be9841d7b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/12aefb2dbb5640598107618a7c6e0fcf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/b6740a1aceef460db1f859381e9eb7ca, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/39d2c9ee5557461c9077cbed167b19f4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1f17b27fc8f546ec94a409c983891f25] to archive 2024-12-02T05:17:10,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:17:10,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/3e86bad8268a46d38947f761d74a2479 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/3e86bad8268a46d38947f761d74a2479 2024-12-02T05:17:10,529 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/c2fb629c636941eea45b18bbb7cba694 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/c2fb629c636941eea45b18bbb7cba694 2024-12-02T05:17:10,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/a1677ea89e7846deae68fbfb8dd06658 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/a1677ea89e7846deae68fbfb8dd06658 2024-12-02T05:17:10,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/fab0afbf8ddb4098bd2bc1eab55d4e12 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/fab0afbf8ddb4098bd2bc1eab55d4e12 2024-12-02T05:17:10,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/dc26811fa1f44683bc6af87696122fa9 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/dc26811fa1f44683bc6af87696122fa9 2024-12-02T05:17:10,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/7aa9408897ab415f922e671853f6f5de to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/7aa9408897ab415f922e671853f6f5de 2024-12-02T05:17:10,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/79ae191becae493cb17ef12b942371ff to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/79ae191becae493cb17ef12b942371ff 2024-12-02T05:17:10,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f1b2ed3e9a34428097a1d73ffe5ad0fa to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f1b2ed3e9a34428097a1d73ffe5ad0fa 2024-12-02T05:17:10,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/8b84a59768c14bc9880ab6a697231f7b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/8b84a59768c14bc9880ab6a697231f7b 2024-12-02T05:17:10,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/51a8c32b23ff4172a386f7fce00e7d53 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/51a8c32b23ff4172a386f7fce00e7d53 2024-12-02T05:17:10,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/c31abbbd27e94fa0ade31b7665e721f4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/c31abbbd27e94fa0ade31b7665e721f4 2024-12-02T05:17:10,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/7fc33ff89037455dacd0e8cac1f47934 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/7fc33ff89037455dacd0e8cac1f47934 2024-12-02T05:17:10,542 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/084bfbdecc864749ae061f35a22ac02a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/084bfbdecc864749ae061f35a22ac02a 2024-12-02T05:17:10,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/5d95780d5f9f480d89adfa56f99d1d14 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/5d95780d5f9f480d89adfa56f99d1d14 2024-12-02T05:17:10,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/6b65257b4c1b45f0a586bc52a06ec8b1 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/6b65257b4c1b45f0a586bc52a06ec8b1 2024-12-02T05:17:10,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/6e2c0deb9b554783ab0c3f49def4c8ec to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/6e2c0deb9b554783ab0c3f49def4c8ec 2024-12-02T05:17:10,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/d352e1c6250c48199eb6d93075a6eb4a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/d352e1c6250c48199eb6d93075a6eb4a 2024-12-02T05:17:10,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f38b9be2511644698b806c259ac4e0e3 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f38b9be2511644698b806c259ac4e0e3 2024-12-02T05:17:10,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/13dc21b52e8742aca44f8b917bff1b32 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/13dc21b52e8742aca44f8b917bff1b32 2024-12-02T05:17:10,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1acb1fc077914b959e93ff39167b6575 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1acb1fc077914b959e93ff39167b6575 2024-12-02T05:17:10,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/ad1608d2586a4bcab88ed84736b27aa3 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/ad1608d2586a4bcab88ed84736b27aa3 2024-12-02T05:17:10,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/a64eaf41bc814da291ffe2ebbe2f8238 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/a64eaf41bc814da291ffe2ebbe2f8238 2024-12-02T05:17:10,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/38cfd27e776742ce9c882ad7ae2c5051 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/38cfd27e776742ce9c882ad7ae2c5051 2024-12-02T05:17:10,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/bb8eb7b4936b4fe4ae6215d9a7661434 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/bb8eb7b4936b4fe4ae6215d9a7661434 2024-12-02T05:17:10,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1d4d2fcff2b1423d8d15f39726b6e9fb to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1d4d2fcff2b1423d8d15f39726b6e9fb 2024-12-02T05:17:10,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1c7971580a694cb18b7c0d996a68b2c5 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1c7971580a694cb18b7c0d996a68b2c5 2024-12-02T05:17:10,562 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f80a52f383dc441ba991bd7e6ab8e5ad to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/f80a52f383dc441ba991bd7e6ab8e5ad 2024-12-02T05:17:10,563 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/2a5735c70efb4af4b5fc174594abdd19 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/2a5735c70efb4af4b5fc174594abdd19 2024-12-02T05:17:10,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/15e8ca9cb39a4df39429f04be9841d7b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/15e8ca9cb39a4df39429f04be9841d7b 2024-12-02T05:17:10,566 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/12aefb2dbb5640598107618a7c6e0fcf to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/12aefb2dbb5640598107618a7c6e0fcf 2024-12-02T05:17:10,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/b6740a1aceef460db1f859381e9eb7ca to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/b6740a1aceef460db1f859381e9eb7ca 2024-12-02T05:17:10,569 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/39d2c9ee5557461c9077cbed167b19f4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/39d2c9ee5557461c9077cbed167b19f4 2024-12-02T05:17:10,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1f17b27fc8f546ec94a409c983891f25 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/1f17b27fc8f546ec94a409c983891f25 2024-12-02T05:17:10,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/9f81768618b449458ede2c7f9d7b2a41, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b9df6c5f613e4a2ca232ae87d309b94b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b77c738d2771445a97cb39013f70a8a5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/8c1cbcbcd25349c681a61a9a5046b69d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/7b78951087ea43f685ad4164b3cc26e9, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/144c1e4c1cde4117918facadf33cf626, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/cb1860a05fec4aaba727da585a63f277, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/5bef9c19cd954ccaba8ad97b47134b3b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b9393e79317e46b7a480192ee37ecd33, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/ff8077f01f37472aae6a2b47234737b6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/378ab6b983eb43abace43a9481bf2793, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/52abf69fcc334fe894e765756be56c4d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0a219f307b334220a02d6cc35c7b9376, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/3670379f92f34703a4c5d9fe4e9eaaf1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/02344fb6a52f4d908356128e9f824691, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/03dd99c954a54f26949b8846c95ea365, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/29cbcefdf876497f918e020f1fff82d1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/98da2875b78c4638afb6fbadf4d14984, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/e2effde792d745f7805de88b80780c03, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/8fa0189a502d460c91b3f7f8bc7efa95, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/88d9006e0d254a0f9591040fe5ac86a3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0d5d40a2a0bf46629e40d9e28fd698dc, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/2ecf65c2b998453b83cfe208a390202d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0c54574779b149958d10d8fd6786cf85, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/3eb29575e11549ed9fd347b49b0db89e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/67e87ea1ecae401c8a3893e34629ed59, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/381d8759ee094154a51528200584c8ed, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/dd259089a25e48ac8d1fa48a4a86f199, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/38f359e226a44fba8a6125d4f8756924, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/12693dd73bbd4ceeb81dafa83b40c1fa, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/59a8b4e63e964f23ac3dbcc3911e3017, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/4e5e514f57f84827b74be099d6bdffcd, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/11ecbd97c4b3416197f7637e23295d98] to archive 2024-12-02T05:17:10,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:17:10,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/9f81768618b449458ede2c7f9d7b2a41 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/9f81768618b449458ede2c7f9d7b2a41 2024-12-02T05:17:10,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b9df6c5f613e4a2ca232ae87d309b94b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b9df6c5f613e4a2ca232ae87d309b94b 2024-12-02T05:17:10,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b77c738d2771445a97cb39013f70a8a5 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b77c738d2771445a97cb39013f70a8a5 2024-12-02T05:17:10,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/8c1cbcbcd25349c681a61a9a5046b69d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/8c1cbcbcd25349c681a61a9a5046b69d 2024-12-02T05:17:10,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/7b78951087ea43f685ad4164b3cc26e9 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/7b78951087ea43f685ad4164b3cc26e9 2024-12-02T05:17:10,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/144c1e4c1cde4117918facadf33cf626 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/144c1e4c1cde4117918facadf33cf626 2024-12-02T05:17:10,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/cb1860a05fec4aaba727da585a63f277 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/cb1860a05fec4aaba727da585a63f277 2024-12-02T05:17:10,594 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/5bef9c19cd954ccaba8ad97b47134b3b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/5bef9c19cd954ccaba8ad97b47134b3b 2024-12-02T05:17:10,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b9393e79317e46b7a480192ee37ecd33 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/b9393e79317e46b7a480192ee37ecd33 2024-12-02T05:17:10,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/ff8077f01f37472aae6a2b47234737b6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/ff8077f01f37472aae6a2b47234737b6 2024-12-02T05:17:10,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/378ab6b983eb43abace43a9481bf2793 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/378ab6b983eb43abace43a9481bf2793 2024-12-02T05:17:10,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/52abf69fcc334fe894e765756be56c4d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/52abf69fcc334fe894e765756be56c4d 2024-12-02T05:17:10,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0a219f307b334220a02d6cc35c7b9376 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0a219f307b334220a02d6cc35c7b9376 2024-12-02T05:17:10,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/3670379f92f34703a4c5d9fe4e9eaaf1 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/3670379f92f34703a4c5d9fe4e9eaaf1 2024-12-02T05:17:10,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/02344fb6a52f4d908356128e9f824691 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/02344fb6a52f4d908356128e9f824691 2024-12-02T05:17:10,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/03dd99c954a54f26949b8846c95ea365 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/03dd99c954a54f26949b8846c95ea365 2024-12-02T05:17:10,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/29cbcefdf876497f918e020f1fff82d1 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/29cbcefdf876497f918e020f1fff82d1 2024-12-02T05:17:10,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/98da2875b78c4638afb6fbadf4d14984 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/98da2875b78c4638afb6fbadf4d14984 2024-12-02T05:17:10,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/e2effde792d745f7805de88b80780c03 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/e2effde792d745f7805de88b80780c03 2024-12-02T05:17:10,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/8fa0189a502d460c91b3f7f8bc7efa95 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/8fa0189a502d460c91b3f7f8bc7efa95 2024-12-02T05:17:10,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/88d9006e0d254a0f9591040fe5ac86a3 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/88d9006e0d254a0f9591040fe5ac86a3 2024-12-02T05:17:10,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0d5d40a2a0bf46629e40d9e28fd698dc to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0d5d40a2a0bf46629e40d9e28fd698dc 2024-12-02T05:17:10,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/2ecf65c2b998453b83cfe208a390202d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/2ecf65c2b998453b83cfe208a390202d 2024-12-02T05:17:10,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0c54574779b149958d10d8fd6786cf85 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/0c54574779b149958d10d8fd6786cf85 2024-12-02T05:17:10,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/3eb29575e11549ed9fd347b49b0db89e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/3eb29575e11549ed9fd347b49b0db89e 2024-12-02T05:17:10,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/67e87ea1ecae401c8a3893e34629ed59 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/67e87ea1ecae401c8a3893e34629ed59 2024-12-02T05:17:10,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/381d8759ee094154a51528200584c8ed to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/381d8759ee094154a51528200584c8ed 2024-12-02T05:17:10,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/dd259089a25e48ac8d1fa48a4a86f199 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/dd259089a25e48ac8d1fa48a4a86f199 2024-12-02T05:17:10,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/38f359e226a44fba8a6125d4f8756924 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/38f359e226a44fba8a6125d4f8756924 2024-12-02T05:17:10,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/12693dd73bbd4ceeb81dafa83b40c1fa to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/12693dd73bbd4ceeb81dafa83b40c1fa 2024-12-02T05:17:10,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/59a8b4e63e964f23ac3dbcc3911e3017 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/59a8b4e63e964f23ac3dbcc3911e3017 2024-12-02T05:17:10,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/4e5e514f57f84827b74be099d6bdffcd to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/4e5e514f57f84827b74be099d6bdffcd 2024-12-02T05:17:10,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/11ecbd97c4b3416197f7637e23295d98 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/11ecbd97c4b3416197f7637e23295d98 2024-12-02T05:17:10,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d6cbed32e7d94883beb6e670c8efff0f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/43c0846d88c04cf180d7e17432b7d926, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5481fbbf5e224c96bfc569098859753b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/760d1fd9859d4d5daa257a67ca436bc3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d48089a3d10c41fc8b56df25200ce783, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/0b8c98a8e8a643e6939de4bad55a37f2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9ddb7c6a74de479d91cf96ee2e98cb09, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/1cc4069eff3247dd8d4c2a0309f0dd9c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/61dc9ed247c24f74897c73c19b0a58d5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/08fc7e64b4b848e297b738ed4f114569, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/7bbfacceddc04014ac569c45aec895de, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/18f13124437c40a0ae07b623abedb367, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/a4afa3593c5c41e9aad9a02efe26734d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/e4001077aca245a7bbd10f1bd606036a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/660158e951014a25923d9f7bc71dbdc4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/4664693c01f34495b5f9c72ded866ed9, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9aff99c158994dc8983ec7df8e1423b0, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9f2cd475af274fbfbcd30502c9a82038, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/b30f3baf9ec6420cb7c6d72bfa4b5b8d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/978623442ed64ab09c055684441df19d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/54e283e37f2f417280e02f90de78857f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/874cc2ff6ca64416b1657b9214cc7ea1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/e9a2773b278648d0b67adb2ef54bbb89, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5312e119ddaf48a1bba26a4fb86d9b51, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/b27ec09898ba4b7eb30d7e4b07298236, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/0875d1c3ac3f4876bed6ec62c7e5352f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5464fd0ed33b4f12b7bf0dc52da63bc8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d689fa0062dc4f7ea99586de6ba2c06d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/67fe5645c1ef4669a9842f60b04af4cf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/84547d042c204df9bc87eedf20ba2f47, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/25f31b1300f1466d90e3f61ab0a6cb69, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/abf4b838abf54a4b8fc685b76551ab3b] to archive 2024-12-02T05:17:10,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:17:10,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d6cbed32e7d94883beb6e670c8efff0f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d6cbed32e7d94883beb6e670c8efff0f 2024-12-02T05:17:10,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/43c0846d88c04cf180d7e17432b7d926 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/43c0846d88c04cf180d7e17432b7d926 2024-12-02T05:17:10,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5481fbbf5e224c96bfc569098859753b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5481fbbf5e224c96bfc569098859753b 2024-12-02T05:17:10,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/760d1fd9859d4d5daa257a67ca436bc3 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/760d1fd9859d4d5daa257a67ca436bc3 2024-12-02T05:17:10,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d48089a3d10c41fc8b56df25200ce783 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d48089a3d10c41fc8b56df25200ce783 2024-12-02T05:17:10,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/0b8c98a8e8a643e6939de4bad55a37f2 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/0b8c98a8e8a643e6939de4bad55a37f2 2024-12-02T05:17:10,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9ddb7c6a74de479d91cf96ee2e98cb09 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9ddb7c6a74de479d91cf96ee2e98cb09 2024-12-02T05:17:10,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/1cc4069eff3247dd8d4c2a0309f0dd9c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/1cc4069eff3247dd8d4c2a0309f0dd9c 2024-12-02T05:17:10,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/61dc9ed247c24f74897c73c19b0a58d5 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/61dc9ed247c24f74897c73c19b0a58d5 2024-12-02T05:17:10,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/08fc7e64b4b848e297b738ed4f114569 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/08fc7e64b4b848e297b738ed4f114569 2024-12-02T05:17:10,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/7bbfacceddc04014ac569c45aec895de to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/7bbfacceddc04014ac569c45aec895de 2024-12-02T05:17:10,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/18f13124437c40a0ae07b623abedb367 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/18f13124437c40a0ae07b623abedb367 2024-12-02T05:17:10,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/a4afa3593c5c41e9aad9a02efe26734d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/a4afa3593c5c41e9aad9a02efe26734d 2024-12-02T05:17:10,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/e4001077aca245a7bbd10f1bd606036a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/e4001077aca245a7bbd10f1bd606036a 2024-12-02T05:17:10,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/660158e951014a25923d9f7bc71dbdc4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/660158e951014a25923d9f7bc71dbdc4 2024-12-02T05:17:10,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/4664693c01f34495b5f9c72ded866ed9 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/4664693c01f34495b5f9c72ded866ed9 2024-12-02T05:17:10,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9aff99c158994dc8983ec7df8e1423b0 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9aff99c158994dc8983ec7df8e1423b0 2024-12-02T05:17:10,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9f2cd475af274fbfbcd30502c9a82038 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/9f2cd475af274fbfbcd30502c9a82038 2024-12-02T05:17:10,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/b30f3baf9ec6420cb7c6d72bfa4b5b8d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/b30f3baf9ec6420cb7c6d72bfa4b5b8d 2024-12-02T05:17:10,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/978623442ed64ab09c055684441df19d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/978623442ed64ab09c055684441df19d 2024-12-02T05:17:10,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/54e283e37f2f417280e02f90de78857f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/54e283e37f2f417280e02f90de78857f 2024-12-02T05:17:10,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/874cc2ff6ca64416b1657b9214cc7ea1 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/874cc2ff6ca64416b1657b9214cc7ea1 2024-12-02T05:17:10,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/e9a2773b278648d0b67adb2ef54bbb89 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/e9a2773b278648d0b67adb2ef54bbb89 2024-12-02T05:17:10,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5312e119ddaf48a1bba26a4fb86d9b51 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5312e119ddaf48a1bba26a4fb86d9b51 2024-12-02T05:17:10,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/b27ec09898ba4b7eb30d7e4b07298236 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/b27ec09898ba4b7eb30d7e4b07298236 2024-12-02T05:17:10,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/0875d1c3ac3f4876bed6ec62c7e5352f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/0875d1c3ac3f4876bed6ec62c7e5352f 2024-12-02T05:17:10,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5464fd0ed33b4f12b7bf0dc52da63bc8 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/5464fd0ed33b4f12b7bf0dc52da63bc8 2024-12-02T05:17:10,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d689fa0062dc4f7ea99586de6ba2c06d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/d689fa0062dc4f7ea99586de6ba2c06d 2024-12-02T05:17:10,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/67fe5645c1ef4669a9842f60b04af4cf to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/67fe5645c1ef4669a9842f60b04af4cf 2024-12-02T05:17:10,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/84547d042c204df9bc87eedf20ba2f47 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/84547d042c204df9bc87eedf20ba2f47 2024-12-02T05:17:10,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/25f31b1300f1466d90e3f61ab0a6cb69 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/25f31b1300f1466d90e3f61ab0a6cb69 2024-12-02T05:17:10,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/abf4b838abf54a4b8fc685b76551ab3b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/abf4b838abf54a4b8fc685b76551ab3b 2024-12-02T05:17:10,667 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/recovered.edits/515.seqid, newMaxSeqId=515, maxSeqId=1 2024-12-02T05:17:10,669 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. 2024-12-02T05:17:10,670 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for d2d081cdce71d4cc4bebd39e9655d288: Waiting for close lock at 1733116629235Running coprocessor pre-close hooks at 1733116629235Disabling compacts and flushes for region at 1733116629235Disabling writes for close at 1733116629235Obtaining lock to block concurrent updates at 1733116629236 (+1 ms)Preparing flush snapshotting stores in d2d081cdce71d4cc4bebd39e9655d288 at 1733116629236Finished memstore snapshotting TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733116629236Flushing stores of TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288. at 1733116629237 (+1 ms)Flushing d2d081cdce71d4cc4bebd39e9655d288/A: creating writer at 1733116629237Flushing d2d081cdce71d4cc4bebd39e9655d288/A: appending metadata at 1733116629240 (+3 ms)Flushing d2d081cdce71d4cc4bebd39e9655d288/A: closing flushed file at 1733116629240Flushing d2d081cdce71d4cc4bebd39e9655d288/B: creating writer at 1733116629656 (+416 ms)Flushing d2d081cdce71d4cc4bebd39e9655d288/B: appending metadata at 1733116629660 (+4 ms)Flushing d2d081cdce71d4cc4bebd39e9655d288/B: closing flushed file at 1733116629660Flushing d2d081cdce71d4cc4bebd39e9655d288/C: creating writer at 1733116630076 (+416 ms)Flushing d2d081cdce71d4cc4bebd39e9655d288/C: appending metadata at 1733116630080 (+4 ms)Flushing d2d081cdce71d4cc4bebd39e9655d288/C: closing flushed file at 1733116630080Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@176f8615: reopening flushed file at 1733116630495 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59f04ec8: reopening flushed file at 1733116630503 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a8db9be: reopening flushed file at 1733116630510 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d2d081cdce71d4cc4bebd39e9655d288 in 1282ms, sequenceid=512, compaction requested=false at 1733116630518 (+8 ms)Writing region close event to WAL at 1733116630663 (+145 ms)Running coprocessor post-close hooks at 1733116630668 (+5 ms)Closed at 1733116630669 (+1 ms) 2024-12-02T05:17:10,672 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:17:10,673 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=d2d081cdce71d4cc4bebd39e9655d288, regionState=CLOSED 2024-12-02T05:17:10,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:17:10,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-02T05:17:10,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure d2d081cdce71d4cc4bebd39e9655d288, server=f49461030c22,46495,1733116594459 in 1.5970 sec 2024-12-02T05:17:10,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-02T05:17:10,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2d081cdce71d4cc4bebd39e9655d288, UNASSIGN in 1.6050 sec 2024-12-02T05:17:10,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-02T05:17:10,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6120 sec 2024-12-02T05:17:10,683 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116630683"}]},"ts":"1733116630683"} 2024-12-02T05:17:10,684 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T05:17:10,684 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T05:17:10,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6250 sec 2024-12-02T05:17:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T05:17:11,218 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:11,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T05:17:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:11,231 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T05:17:11,233 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:11,239 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:17:11,243 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/recovered.edits] 2024-12-02T05:17:11,246 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/bdd245916c0047e09ea18f875f508fd5 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/bdd245916c0047e09ea18f875f508fd5 2024-12-02T05:17:11,248 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/dd81367304f04df486d26af40398054f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/A/dd81367304f04df486d26af40398054f 2024-12-02T05:17:11,250 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/4e873b98e1084d1c9f159a23bd9d1cb4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/4e873b98e1084d1c9f159a23bd9d1cb4 2024-12-02T05:17:11,251 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/fc9645751fb845098bd1207edfc76394 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/B/fc9645751fb845098bd1207edfc76394 2024-12-02T05:17:11,254 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/53deb584fb7a476bac0d220df6144494 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/53deb584fb7a476bac0d220df6144494 2024-12-02T05:17:11,255 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/edb62086370446c4a9b92f0556bfcc35 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/C/edb62086370446c4a9b92f0556bfcc35 2024-12-02T05:17:11,258 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/recovered.edits/515.seqid to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288/recovered.edits/515.seqid 2024-12-02T05:17:11,259 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/d2d081cdce71d4cc4bebd39e9655d288 2024-12-02T05:17:11,259 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T05:17:11,264 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:11,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-02T05:17:11,273 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T05:17:11,299 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T05:17:11,300 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:11,300 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T05:17:11,300 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733116631300"}]},"ts":"9223372036854775807"} 2024-12-02T05:17:11,305 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T05:17:11,305 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d2d081cdce71d4cc4bebd39e9655d288, NAME => 'TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T05:17:11,305 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T05:17:11,305 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733116631305"}]},"ts":"9223372036854775807"} 2024-12-02T05:17:11,308 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T05:17:11,309 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:11,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 84 msec 2024-12-02T05:17:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T05:17:11,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T05:17:11,338 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:11,350 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/f49461030c22:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/f49461030c22:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;f49461030c22:46495-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_359649851_22 at /127.0.0.1:46004 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=588 (was 477) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5989 (was 6579) 2024-12-02T05:17:11,361 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=588, ProcessCount=11, AvailableMemoryMB=5987 2024-12-02T05:17:11,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T05:17:11,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T05:17:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:11,365 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T05:17:11,365 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:11,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-12-02T05:17:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-02T05:17:11,366 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T05:17:11,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741941_1117 (size=1061) 2024-12-02T05:17:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-02T05:17:11,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-02T05:17:11,777 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c 2024-12-02T05:17:11,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741942_1118 (size=53) 2024-12-02T05:17:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-02T05:17:12,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:17:12,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ffa3b22adefa33dc88a53908588e6f34, disabling compactions & flushes 2024-12-02T05:17:12,186 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:12,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:12,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. after waiting 0 ms 2024-12-02T05:17:12,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:12,186 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:12,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ffa3b22adefa33dc88a53908588e6f34: Waiting for close lock at 1733116632186Disabling compacts and flushes for region at 1733116632186Disabling writes for close at 1733116632186Writing region close event to WAL at 1733116632186Closed at 1733116632186 2024-12-02T05:17:12,188 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T05:17:12,189 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733116632188"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733116632188"}]},"ts":"1733116632188"} 2024-12-02T05:17:12,191 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T05:17:12,193 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T05:17:12,193 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116632193"}]},"ts":"1733116632193"} 2024-12-02T05:17:12,195 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T05:17:12,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffa3b22adefa33dc88a53908588e6f34, ASSIGN}] 2024-12-02T05:17:12,198 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffa3b22adefa33dc88a53908588e6f34, ASSIGN 2024-12-02T05:17:12,199 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffa3b22adefa33dc88a53908588e6f34, ASSIGN; state=OFFLINE, location=f49461030c22,46495,1733116594459; forceNewPlan=false, retain=false 2024-12-02T05:17:12,350 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=ffa3b22adefa33dc88a53908588e6f34, regionState=OPENING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:17:12,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffa3b22adefa33dc88a53908588e6f34, ASSIGN because future has completed 2024-12-02T05:17:12,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:17:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-02T05:17:12,510 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:12,510 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} 2024-12-02T05:17:12,511 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:12,511 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:17:12,511 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:12,511 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:12,513 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:12,515 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:17:12,516 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffa3b22adefa33dc88a53908588e6f34 columnFamilyName A 2024-12-02T05:17:12,516 DEBUG [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:12,517 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.HStore(327): Store=ffa3b22adefa33dc88a53908588e6f34/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:17:12,518 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:12,519 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:17:12,520 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffa3b22adefa33dc88a53908588e6f34 columnFamilyName B 2024-12-02T05:17:12,520 DEBUG [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:12,521 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.HStore(327): Store=ffa3b22adefa33dc88a53908588e6f34/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:17:12,521 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:12,523 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:17:12,523 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffa3b22adefa33dc88a53908588e6f34 columnFamilyName C 2024-12-02T05:17:12,523 DEBUG [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:12,524 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.HStore(327): Store=ffa3b22adefa33dc88a53908588e6f34/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:17:12,524 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:12,524 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:12,525 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:12,525 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:12,526 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:12,526 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:12,527 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T05:17:12,528 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:12,531 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T05:17:12,531 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened ffa3b22adefa33dc88a53908588e6f34; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67988776, jitterRate=0.013111710548400879}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T05:17:12,531 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:12,532 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for ffa3b22adefa33dc88a53908588e6f34: Running coprocessor pre-open hook at 1733116632511Writing region info on filesystem at 1733116632511Initializing all the Stores at 1733116632512 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116632512Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116632512Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116632512Cleaning up temporary data from old regions at 1733116632526 (+14 ms)Running coprocessor post-open hooks at 1733116632531 (+5 ms)Region opened successfully at 1733116632532 (+1 ms) 2024-12-02T05:17:12,533 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., pid=32, masterSystemTime=1733116632507 2024-12-02T05:17:12,535 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:12,536 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:12,536 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=ffa3b22adefa33dc88a53908588e6f34, regionState=OPEN, openSeqNum=2, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:17:12,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:17:12,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-02T05:17:12,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 in 185 msec 2024-12-02T05:17:12,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-02T05:17:12,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffa3b22adefa33dc88a53908588e6f34, ASSIGN in 345 msec 2024-12-02T05:17:12,544 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T05:17:12,544 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116632544"}]},"ts":"1733116632544"} 2024-12-02T05:17:12,546 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T05:17:12,547 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T05:17:12,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-12-02T05:17:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-02T05:17:13,509 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:13,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:13,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:13,512 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:13,515 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f15291f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:13,520 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@230c8e56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:13,521 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:13,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:13,525 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T05:17:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is f49461030c22,42731,1733116593060 2024-12-02T05:17:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@50990c06 2024-12-02T05:17:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T05:17:13,527 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T05:17:13,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T05:17:13,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T05:17:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:13,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741943_1119 (size=1097) 2024-12-02T05:17:14,024 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T05:17:14,024 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T05:17:14,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T05:17:14,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffa3b22adefa33dc88a53908588e6f34, REOPEN/MOVE}] 2024-12-02T05:17:14,037 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffa3b22adefa33dc88a53908588e6f34, REOPEN/MOVE 2024-12-02T05:17:14,038 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=ffa3b22adefa33dc88a53908588e6f34, regionState=CLOSING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:17:14,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffa3b22adefa33dc88a53908588e6f34, REOPEN/MOVE because future has completed 2024-12-02T05:17:14,040 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T05:17:14,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:17:14,193 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,193 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T05:17:14,193 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing ffa3b22adefa33dc88a53908588e6f34, disabling compactions & flushes 2024-12-02T05:17:14,193 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:14,193 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:14,193 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. after waiting 0 ms 2024-12-02T05:17:14,193 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:14,197 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T05:17:14,198 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:14,198 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for ffa3b22adefa33dc88a53908588e6f34: Waiting for close lock at 1733116634193Running coprocessor pre-close hooks at 1733116634193Disabling compacts and flushes for region at 1733116634193Disabling writes for close at 1733116634193Writing region close event to WAL at 1733116634193Running coprocessor post-close hooks at 1733116634198 (+5 ms)Closed at 1733116634198 2024-12-02T05:17:14,198 WARN [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: ffa3b22adefa33dc88a53908588e6f34 to self. 2024-12-02T05:17:14,200 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,201 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=ffa3b22adefa33dc88a53908588e6f34, regionState=CLOSED 2024-12-02T05:17:14,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:17:14,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-02T05:17:14,206 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 in 164 msec 2024-12-02T05:17:14,206 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffa3b22adefa33dc88a53908588e6f34, REOPEN/MOVE; state=CLOSED, location=f49461030c22,46495,1733116594459; forceNewPlan=false, retain=true 2024-12-02T05:17:14,357 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=ffa3b22adefa33dc88a53908588e6f34, regionState=OPENING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:17:14,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffa3b22adefa33dc88a53908588e6f34, REOPEN/MOVE because future has completed 2024-12-02T05:17:14,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:17:14,534 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:14,534 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} 2024-12-02T05:17:14,535 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,535 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:17:14,535 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,535 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,542 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,544 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:17:14,552 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffa3b22adefa33dc88a53908588e6f34 columnFamilyName A 2024-12-02T05:17:14,554 DEBUG [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:14,555 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.HStore(327): Store=ffa3b22adefa33dc88a53908588e6f34/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:17:14,555 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,556 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:17:14,556 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffa3b22adefa33dc88a53908588e6f34 columnFamilyName B 2024-12-02T05:17:14,556 DEBUG [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:14,557 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.HStore(327): Store=ffa3b22adefa33dc88a53908588e6f34/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:17:14,557 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,558 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:17:14,558 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffa3b22adefa33dc88a53908588e6f34 columnFamilyName C 2024-12-02T05:17:14,558 DEBUG [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:14,558 INFO [StoreOpener-ffa3b22adefa33dc88a53908588e6f34-1 {}] regionserver.HStore(327): Store=ffa3b22adefa33dc88a53908588e6f34/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:17:14,559 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:14,559 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,559 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,560 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,561 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,561 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,562 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T05:17:14,563 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,564 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened ffa3b22adefa33dc88a53908588e6f34; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73735343, jitterRate=0.09874223172664642}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T05:17:14,564 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,566 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for ffa3b22adefa33dc88a53908588e6f34: Running coprocessor pre-open hook at 1733116634535Writing region info on filesystem at 1733116634535Initializing all the Stores at 1733116634536 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116634536Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116634539 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116634539Cleaning up temporary data from old regions at 1733116634561 (+22 ms)Running coprocessor post-open hooks at 1733116634564 (+3 ms)Region opened successfully at 1733116634566 (+2 ms) 2024-12-02T05:17:14,568 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., pid=37, masterSystemTime=1733116634531 2024-12-02T05:17:14,570 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:14,570 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:14,571 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=ffa3b22adefa33dc88a53908588e6f34, regionState=OPEN, openSeqNum=5, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:17:14,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:17:14,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-12-02T05:17:14,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 in 213 msec 2024-12-02T05:17:14,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-02T05:17:14,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffa3b22adefa33dc88a53908588e6f34, REOPEN/MOVE in 540 msec 2024-12-02T05:17:14,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-02T05:17:14,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 552 msec 2024-12-02T05:17:14,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0500 sec 2024-12-02T05:17:14,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-02T05:17:14,589 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:14,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:14,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:14,594 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53dbdaac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:14,596 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:14,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b66406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,601 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:14,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:14,612 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:14,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12dd14a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:14,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:14,618 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac65143, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,624 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:14,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:14,626 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59974273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:14,630 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71040636, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:14,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:14,641 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ac7f41c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:14,644 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dfd7cd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:14,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:14,650 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:14,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:14,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b09bc38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:14,652 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@525fc57c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:14,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:14,668 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a926027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:14,671 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c1fbab6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,677 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:14,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:14,678 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699d3459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:14,681 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1a645b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:14,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:14,694 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@691e4cdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:14,696 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395bd548, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:14,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:14,702 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472901fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:14,704 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:14,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a8b0230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:14,709 DEBUG [Thread-619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:14,711 DEBUG [Thread-619 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:14,712 DEBUG [Thread-619 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:14,713 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:14,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-02T05:17:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-02T05:17:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:17:14,718 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:14,719 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:14,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:14,729 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:14,731 DEBUG [Thread-612 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:14,733 DEBUG [Thread-612 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:14,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:17:14,743 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:14,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:14,751 DEBUG [Thread-621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:17:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:14,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:17:14,759 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:14,762 DEBUG [Thread-623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:14,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:14,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:14,765 DEBUG [Thread-616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:14,766 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:17:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:14,771 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:14,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:14,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:17:14,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:17:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:17:14,781 DEBUG [Thread-608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T05:17:14,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:14,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:14,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:14,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:14,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:14,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:14,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:14,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:17:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-02T05:17:14,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T05:17:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:14,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:14,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d655846101ff4040856b907779e2f3e1_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116634752/Put/seqid=0 2024-12-02T05:17:14,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116634902, max timestamp=1733116634913Num uniques 859; 2024-12-02T05:17:14,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116634902, max timestamp=1733116634915Num uniques 859; 2024-12-02T05:17:14,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:17:14,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:17:14,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116634902, max timestamp=1733116634915Num uniques 859; 2024-12-02T05:17:14,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:17:14,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741944_1120 (size=12154) 2024-12-02T05:17:14,944 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:14,955 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d655846101ff4040856b907779e2f3e1_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d655846101ff4040856b907779e2f3e1_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:14,956 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/e2e71b8487c84eb9b7c69963c24623b1, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:14,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/e2e71b8487c84eb9b7c69963c24623b1 is 175, key is test_row_0/A:col10/1733116634752/Put/seqid=0 2024-12-02T05:17:14,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116694958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:14,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116694960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:14,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116694962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:14,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:14,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116694964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116694966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741945_1121 (size=30955) 2024-12-02T05:17:15,026 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/e2e71b8487c84eb9b7c69963c24623b1 2024-12-02T05:17:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-02T05:17:15,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T05:17:15,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:15,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:15,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:15,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/c24fcdf64f95474eb926b87bb86efd70 is 50, key is test_row_0/B:col10/1733116634752/Put/seqid=0 2024-12-02T05:17:15,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116695078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116695098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116695098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116695099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116695100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741946_1122 (size=12001) 2024-12-02T05:17:15,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T05:17:15,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:15,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:15,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:15,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116695302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116695319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116695319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116695319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116695319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-02T05:17:15,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T05:17:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:15,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T05:17:15,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:15,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:15,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:15,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/c24fcdf64f95474eb926b87bb86efd70 2024-12-02T05:17:15,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/57e766a45b3f4946a96b78617798adbf is 50, key is test_row_0/C:col10/1733116634752/Put/seqid=0 2024-12-02T05:17:15,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741947_1123 (size=12001) 2024-12-02T05:17:15,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/57e766a45b3f4946a96b78617798adbf 2024-12-02T05:17:15,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/e2e71b8487c84eb9b7c69963c24623b1 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e2e71b8487c84eb9b7c69963c24623b1 2024-12-02T05:17:15,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e2e71b8487c84eb9b7c69963c24623b1, entries=150, sequenceid=14, filesize=30.2 K 2024-12-02T05:17:15,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/c24fcdf64f95474eb926b87bb86efd70 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/c24fcdf64f95474eb926b87bb86efd70 2024-12-02T05:17:15,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/c24fcdf64f95474eb926b87bb86efd70, entries=150, sequenceid=14, filesize=11.7 K 2024-12-02T05:17:15,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/57e766a45b3f4946a96b78617798adbf as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/57e766a45b3f4946a96b78617798adbf 2024-12-02T05:17:15,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/57e766a45b3f4946a96b78617798adbf, entries=150, sequenceid=14, filesize=11.7 K 2024-12-02T05:17:15,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ffa3b22adefa33dc88a53908588e6f34 in 816ms, sequenceid=14, compaction requested=false 2024-12-02T05:17:15,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:15,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-02T05:17:15,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:15,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T05:17:15,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:15,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T05:17:15,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:15,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T05:17:15,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202da3892d6862f4aaba761b2196a59c048_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116634961/Put/seqid=0 2024-12-02T05:17:15,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116695631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116695635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116695652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T05:17:15,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:15,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:15,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:15,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116695663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116695663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741948_1124 (size=14594) 2024-12-02T05:17:15,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116695768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116695768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116695769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116695778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116695778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T05:17:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:15,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:15,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:15,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-02T05:17:15,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116695978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116695978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116695978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T05:17:15,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:15,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:15,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:15,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:15,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116695987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:15,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116695997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,089 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:16,093 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202da3892d6862f4aaba761b2196a59c048_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202da3892d6862f4aaba761b2196a59c048_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:16,094 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/829c72fcab5d45b5b0057b41b736a850, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:16,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/829c72fcab5d45b5b0057b41b736a850 is 175, key is test_row_0/A:col10/1733116634961/Put/seqid=0 2024-12-02T05:17:16,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741949_1125 (size=39549) 2024-12-02T05:17:16,101 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/829c72fcab5d45b5b0057b41b736a850 2024-12-02T05:17:16,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/222dc9dcc5d145cd866cc6fb832d9f68 is 50, key is test_row_0/B:col10/1733116634961/Put/seqid=0 2024-12-02T05:17:16,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741950_1126 (size=12001) 2024-12-02T05:17:16,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/222dc9dcc5d145cd866cc6fb832d9f68 2024-12-02T05:17:16,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T05:17:16,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:16,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:16,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:16,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:16,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:16,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/e48fd93db5ea4d30ba4022bd9e8282a6 is 50, key is test_row_0/C:col10/1733116634961/Put/seqid=0 2024-12-02T05:17:16,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741951_1127 (size=12001) 2024-12-02T05:17:16,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/e48fd93db5ea4d30ba4022bd9e8282a6 2024-12-02T05:17:16,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/829c72fcab5d45b5b0057b41b736a850 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/829c72fcab5d45b5b0057b41b736a850 2024-12-02T05:17:16,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/829c72fcab5d45b5b0057b41b736a850, entries=200, sequenceid=52, filesize=38.6 K 2024-12-02T05:17:16,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/222dc9dcc5d145cd866cc6fb832d9f68 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/222dc9dcc5d145cd866cc6fb832d9f68 2024-12-02T05:17:16,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/222dc9dcc5d145cd866cc6fb832d9f68, entries=150, sequenceid=52, filesize=11.7 K 2024-12-02T05:17:16,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/e48fd93db5ea4d30ba4022bd9e8282a6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/e48fd93db5ea4d30ba4022bd9e8282a6 2024-12-02T05:17:16,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/e48fd93db5ea4d30ba4022bd9e8282a6, entries=150, sequenceid=52, filesize=11.7 K 2024-12-02T05:17:16,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=53.67 KB/54960 for ffa3b22adefa33dc88a53908588e6f34 in 584ms, sequenceid=52, compaction requested=false 2024-12-02T05:17:16,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:16,216 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T05:17:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:16,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T05:17:16,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:16,291 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T05:17:16,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:16,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:16,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:16,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:16,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:16,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:16,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:16,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024feec190ce9b48178a604c5baf3c4300_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116636290/Put/seqid=0 2024-12-02T05:17:16,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116696336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116696336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116696336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116696338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116696338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741952_1128 (size=12154) 2024-12-02T05:17:16,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:16,365 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024feec190ce9b48178a604c5baf3c4300_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024feec190ce9b48178a604c5baf3c4300_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:16,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/11644da6078747bebee67a856e9cf342, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:16,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/11644da6078747bebee67a856e9cf342 is 175, key is test_row_0/A:col10/1733116636290/Put/seqid=0 2024-12-02T05:17:16,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741953_1129 (size=30955) 2024-12-02T05:17:16,393 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=65, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/11644da6078747bebee67a856e9cf342 2024-12-02T05:17:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/2a83930e713f41759f0e247dbc24e319 is 50, key is test_row_0/B:col10/1733116636290/Put/seqid=0 2024-12-02T05:17:16,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741954_1130 (size=12001) 2024-12-02T05:17:16,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116696447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,449 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/2a83930e713f41759f0e247dbc24e319 2024-12-02T05:17:16,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116696459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116696459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116696460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116696460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/f837a19ef7e049b9996ffe4d637da9ca is 50, key is test_row_0/C:col10/1733116636290/Put/seqid=0 2024-12-02T05:17:16,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741955_1131 (size=12001) 2024-12-02T05:17:16,491 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/f837a19ef7e049b9996ffe4d637da9ca 2024-12-02T05:17:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/11644da6078747bebee67a856e9cf342 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/11644da6078747bebee67a856e9cf342 2024-12-02T05:17:16,509 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/11644da6078747bebee67a856e9cf342, entries=150, sequenceid=65, filesize=30.2 K 2024-12-02T05:17:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/2a83930e713f41759f0e247dbc24e319 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/2a83930e713f41759f0e247dbc24e319 2024-12-02T05:17:16,517 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/2a83930e713f41759f0e247dbc24e319, entries=150, sequenceid=65, filesize=11.7 K 2024-12-02T05:17:16,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/f837a19ef7e049b9996ffe4d637da9ca as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/f837a19ef7e049b9996ffe4d637da9ca 2024-12-02T05:17:16,522 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/f837a19ef7e049b9996ffe4d637da9ca, entries=150, sequenceid=65, filesize=11.7 K 2024-12-02T05:17:16,523 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for ffa3b22adefa33dc88a53908588e6f34 in 232ms, sequenceid=65, compaction requested=true 2024-12-02T05:17:16,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:16,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:16,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-02T05:17:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-02T05:17:16,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-02T05:17:16,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8060 sec 2024-12-02T05:17:16,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.8140 sec 2024-12-02T05:17:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:16,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T05:17:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:16,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:16,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116696673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116696675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116696675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116696678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116696678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120207a54ee1411749019b11ad23c896598c_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116636333/Put/seqid=0 2024-12-02T05:17:16,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741956_1132 (size=14594) 2024-12-02T05:17:16,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116696787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116696788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116696788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116696788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-02T05:17:16,868 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:16,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-02T05:17:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T05:17:16,871 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:16,873 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:16,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T05:17:16,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116696989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:16,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116696997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116696998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:16,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116696998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116696998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:17,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:17,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:17,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:17,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:17,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,130 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:17,135 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120207a54ee1411749019b11ad23c896598c_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120207a54ee1411749019b11ad23c896598c_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:17,136 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/e6b2187eb5aa494580090b819ae5dfb6, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:17,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/e6b2187eb5aa494580090b819ae5dfb6 is 175, key is test_row_0/A:col10/1733116636333/Put/seqid=0 2024-12-02T05:17:17,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741957_1133 (size=39549) 2024-12-02T05:17:17,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:17,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:17,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:17,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:17,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T05:17:17,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116697307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:17,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116697307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:17,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116697308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116697308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:17,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:17,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:17,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:17,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:17,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:17,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:17,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:17,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:17,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T05:17:17,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116697497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:17,551 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/e6b2187eb5aa494580090b819ae5dfb6 2024-12-02T05:17:17,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/71300e7ca40e4242add027a827ddb9a6 is 50, key is test_row_0/B:col10/1733116636333/Put/seqid=0 2024-12-02T05:17:17,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741958_1134 (size=12001) 2024-12-02T05:17:17,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:17,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:17,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:17,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116697818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:17,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:17,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:17,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116697818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:17,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:17,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116697818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116697818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:17,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:17,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:17,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:17,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:17,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:17,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/71300e7ca40e4242add027a827ddb9a6 2024-12-02T05:17:17,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/111a517d7ee44e0683eafc8ef4210afe is 50, key is test_row_0/C:col10/1733116636333/Put/seqid=0 2024-12-02T05:17:18,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741959_1135 (size=12001) 2024-12-02T05:17:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T05:17:18,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:18,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:18,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:18,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:18,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/111a517d7ee44e0683eafc8ef4210afe 2024-12-02T05:17:18,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:18,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:18,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/e6b2187eb5aa494580090b819ae5dfb6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e6b2187eb5aa494580090b819ae5dfb6 2024-12-02T05:17:18,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e6b2187eb5aa494580090b819ae5dfb6, entries=200, sequenceid=86, filesize=38.6 K 2024-12-02T05:17:18,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/71300e7ca40e4242add027a827ddb9a6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/71300e7ca40e4242add027a827ddb9a6 2024-12-02T05:17:18,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/71300e7ca40e4242add027a827ddb9a6, entries=150, sequenceid=86, filesize=11.7 K 2024-12-02T05:17:18,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/111a517d7ee44e0683eafc8ef4210afe as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/111a517d7ee44e0683eafc8ef4210afe 2024-12-02T05:17:18,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/111a517d7ee44e0683eafc8ef4210afe, entries=150, sequenceid=86, filesize=11.7 K 2024-12-02T05:17:18,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for ffa3b22adefa33dc88a53908588e6f34 in 1775ms, sequenceid=86, compaction requested=true 2024-12-02T05:17:18,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:18,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffa3b22adefa33dc88a53908588e6f34:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:17:18,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:18,434 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:17:18,434 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:17:18,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffa3b22adefa33dc88a53908588e6f34:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:17:18,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:18,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffa3b22adefa33dc88a53908588e6f34:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:17:18,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:18,437 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:17:18,437 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:17:18,437 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ffa3b22adefa33dc88a53908588e6f34/B is initiating minor compaction (all files) 2024-12-02T05:17:18,437 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): ffa3b22adefa33dc88a53908588e6f34/A is initiating minor compaction (all files) 2024-12-02T05:17:18,437 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffa3b22adefa33dc88a53908588e6f34/A in TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,437 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffa3b22adefa33dc88a53908588e6f34/B in TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,437 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e2e71b8487c84eb9b7c69963c24623b1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/829c72fcab5d45b5b0057b41b736a850, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/11644da6078747bebee67a856e9cf342, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e6b2187eb5aa494580090b819ae5dfb6] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp, totalSize=137.7 K 2024-12-02T05:17:18,437 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/c24fcdf64f95474eb926b87bb86efd70, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/222dc9dcc5d145cd866cc6fb832d9f68, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/2a83930e713f41759f0e247dbc24e319, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/71300e7ca40e4242add027a827ddb9a6] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp, totalSize=46.9 K 2024-12-02T05:17:18,438 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,438 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e2e71b8487c84eb9b7c69963c24623b1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/829c72fcab5d45b5b0057b41b736a850, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/11644da6078747bebee67a856e9cf342, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e6b2187eb5aa494580090b819ae5dfb6] 2024-12-02T05:17:18,438 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting c24fcdf64f95474eb926b87bb86efd70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733116634752 2024-12-02T05:17:18,438 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2e71b8487c84eb9b7c69963c24623b1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733116634752 2024-12-02T05:17:18,439 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 829c72fcab5d45b5b0057b41b736a850, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733116634959 2024-12-02T05:17:18,439 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 222dc9dcc5d145cd866cc6fb832d9f68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733116634959 2024-12-02T05:17:18,439 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11644da6078747bebee67a856e9cf342, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733116635616 2024-12-02T05:17:18,439 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a83930e713f41759f0e247dbc24e319, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733116635616 2024-12-02T05:17:18,440 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 71300e7ca40e4242add027a827ddb9a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733116636315 2024-12-02T05:17:18,440 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6b2187eb5aa494580090b819ae5dfb6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733116636315 2024-12-02T05:17:18,448 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:18,450 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffa3b22adefa33dc88a53908588e6f34#B#compaction#116 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:18,451 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/510dff0b1ba246b891b6512f72cb83ac is 50, key is test_row_0/B:col10/1733116636333/Put/seqid=0 2024-12-02T05:17:18,457 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412022dc99f90513241fb96c77688d1ef1062_ffa3b22adefa33dc88a53908588e6f34 store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:18,460 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412022dc99f90513241fb96c77688d1ef1062_ffa3b22adefa33dc88a53908588e6f34, store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:18,461 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022dc99f90513241fb96c77688d1ef1062_ffa3b22adefa33dc88a53908588e6f34 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:18,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741961_1137 (size=4469) 2024-12-02T05:17:18,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741960_1136 (size=12139) 2024-12-02T05:17:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:18,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T05:17:18,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:18,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d4e445bb8bd74805ba2fd44811a84732_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116638518/Put/seqid=0 2024-12-02T05:17:18,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741962_1138 (size=14594) 2024-12-02T05:17:18,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:18,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:18,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116698601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:18,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:18,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116698717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:18,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:18,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:18,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:18,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:18,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:18,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116698827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:18,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116698827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:18,828 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_0' in ffa3b22adefa33dc88a53908588e6f34 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:18,828 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in ffa3b22adefa33dc88a53908588e6f34 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:18,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116698837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:18,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116698837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:18,842 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in ffa3b22adefa33dc88a53908588e6f34 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:18,842 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in ffa3b22adefa33dc88a53908588e6f34 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:18,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:18,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:18,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:18,875 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffa3b22adefa33dc88a53908588e6f34#A#compaction#117 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:18,876 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/597a7acdaea049fb89474c4859324e32 is 175, key is test_row_0/A:col10/1733116636333/Put/seqid=0 2024-12-02T05:17:18,881 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/510dff0b1ba246b891b6512f72cb83ac as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/510dff0b1ba246b891b6512f72cb83ac 2024-12-02T05:17:18,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741963_1139 (size=31093) 2024-12-02T05:17:18,889 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffa3b22adefa33dc88a53908588e6f34/B of ffa3b22adefa33dc88a53908588e6f34 into 510dff0b1ba246b891b6512f72cb83ac(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:18,889 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:18,889 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., storeName=ffa3b22adefa33dc88a53908588e6f34/B, priority=12, startTime=1733116638434; duration=0sec 2024-12-02T05:17:18,890 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:18,890 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffa3b22adefa33dc88a53908588e6f34:B 2024-12-02T05:17:18,890 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:17:18,891 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:17:18,891 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ffa3b22adefa33dc88a53908588e6f34/C is initiating minor compaction (all files) 2024-12-02T05:17:18,891 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffa3b22adefa33dc88a53908588e6f34/C in TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:18,891 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/57e766a45b3f4946a96b78617798adbf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/e48fd93db5ea4d30ba4022bd9e8282a6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/f837a19ef7e049b9996ffe4d637da9ca, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/111a517d7ee44e0683eafc8ef4210afe] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp, totalSize=46.9 K 2024-12-02T05:17:18,891 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e766a45b3f4946a96b78617798adbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733116634752 2024-12-02T05:17:18,892 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting e48fd93db5ea4d30ba4022bd9e8282a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733116634959 2024-12-02T05:17:18,892 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting f837a19ef7e049b9996ffe4d637da9ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733116635616 2024-12-02T05:17:18,893 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 111a517d7ee44e0683eafc8ef4210afe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733116636315 2024-12-02T05:17:18,913 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffa3b22adefa33dc88a53908588e6f34#C#compaction#119 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:18,914 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/b9799e649e034706bf6e64d161363b6a is 50, key is test_row_0/C:col10/1733116636333/Put/seqid=0 2024-12-02T05:17:18,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116698927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:18,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:18,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:18,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:18,936 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:18,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741964_1140 (size=12139) 2024-12-02T05:17:18,943 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d4e445bb8bd74805ba2fd44811a84732_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d4e445bb8bd74805ba2fd44811a84732_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:18,945 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/1dddeb376ec343afb595c93cc9669663, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:18,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/1dddeb376ec343afb595c93cc9669663 is 175, key is test_row_0/A:col10/1733116638518/Put/seqid=0 2024-12-02T05:17:18,949 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/b9799e649e034706bf6e64d161363b6a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/b9799e649e034706bf6e64d161363b6a 2024-12-02T05:17:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T05:17:19,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:19,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:19,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:19,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:19,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:19,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:19,082 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffa3b22adefa33dc88a53908588e6f34/C of ffa3b22adefa33dc88a53908588e6f34 into b9799e649e034706bf6e64d161363b6a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:19,083 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:19,083 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., storeName=ffa3b22adefa33dc88a53908588e6f34/C, priority=12, startTime=1733116638434; duration=0sec 2024-12-02T05:17:19,083 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:19,083 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffa3b22adefa33dc88a53908588e6f34:C 2024-12-02T05:17:19,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741965_1141 (size=39549) 2024-12-02T05:17:19,090 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/1dddeb376ec343afb595c93cc9669663 2024-12-02T05:17:19,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/5507035f74594910b8d6a07dde7692fa is 50, key is test_row_0/B:col10/1733116638518/Put/seqid=0 2024-12-02T05:17:19,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741966_1142 (size=12001) 2024-12-02T05:17:19,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/5507035f74594910b8d6a07dde7692fa 2024-12-02T05:17:19,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/d012ef187ca14b2ab67029227165f11f is 50, key is test_row_0/C:col10/1733116638518/Put/seqid=0 2024-12-02T05:17:19,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741967_1143 (size=12001) 2024-12-02T05:17:19,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:19,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:19,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:19,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:19,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:19,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:19,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116699237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:19,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:19,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:19,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:19,293 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/597a7acdaea049fb89474c4859324e32 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/597a7acdaea049fb89474c4859324e32 2024-12-02T05:17:19,299 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffa3b22adefa33dc88a53908588e6f34/A of ffa3b22adefa33dc88a53908588e6f34 into 597a7acdaea049fb89474c4859324e32(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:19,299 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:19,299 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., storeName=ffa3b22adefa33dc88a53908588e6f34/A, priority=12, startTime=1733116638434; duration=0sec 2024-12-02T05:17:19,299 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:19,299 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffa3b22adefa33dc88a53908588e6f34:A 2024-12-02T05:17:19,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:19,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:19,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:19,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:19,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:19,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:19,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:19,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:19,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:19,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:19,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:19,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:19,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/d012ef187ca14b2ab67029227165f11f 2024-12-02T05:17:19,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/1dddeb376ec343afb595c93cc9669663 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/1dddeb376ec343afb595c93cc9669663 2024-12-02T05:17:19,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/1dddeb376ec343afb595c93cc9669663, entries=200, sequenceid=98, filesize=38.6 K 2024-12-02T05:17:19,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/5507035f74594910b8d6a07dde7692fa as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/5507035f74594910b8d6a07dde7692fa 2024-12-02T05:17:19,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/5507035f74594910b8d6a07dde7692fa, entries=150, sequenceid=98, filesize=11.7 K 2024-12-02T05:17:19,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/d012ef187ca14b2ab67029227165f11f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/d012ef187ca14b2ab67029227165f11f 2024-12-02T05:17:19,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/d012ef187ca14b2ab67029227165f11f, entries=150, sequenceid=98, filesize=11.7 K 2024-12-02T05:17:19,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ffa3b22adefa33dc88a53908588e6f34 in 1063ms, sequenceid=98, compaction requested=false 2024-12-02T05:17:19,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:19,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T05:17:19,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:19,685 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T05:17:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:19,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202517f39c113a844a9881c8c1466847e9d_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116638596/Put/seqid=0 2024-12-02T05:17:19,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741968_1144 (size=12154) 2024-12-02T05:17:19,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:19,702 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202517f39c113a844a9881c8c1466847e9d_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202517f39c113a844a9881c8c1466847e9d_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:19,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/78061e250519413f8a3076acdcb391bf, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:19,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/78061e250519413f8a3076acdcb391bf is 175, key is test_row_0/A:col10/1733116638596/Put/seqid=0 2024-12-02T05:17:19,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741969_1145 (size=30955) 2024-12-02T05:17:19,708 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/78061e250519413f8a3076acdcb391bf 2024-12-02T05:17:19,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/f88c4bd1e9bc4c42ae18ea13873c4d17 is 50, key is test_row_0/B:col10/1733116638596/Put/seqid=0 2024-12-02T05:17:19,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741970_1146 (size=12001) 2024-12-02T05:17:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:19,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:19,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116699778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:19,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116699887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:20,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116700098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:20,121 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/f88c4bd1e9bc4c42ae18ea13873c4d17 2024-12-02T05:17:20,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/98941203cae047f1a85718e82a3c7bae is 50, key is test_row_0/C:col10/1733116638596/Put/seqid=0 2024-12-02T05:17:20,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741971_1147 (size=12001) 2024-12-02T05:17:20,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116700407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:20,536 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/98941203cae047f1a85718e82a3c7bae 2024-12-02T05:17:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/78061e250519413f8a3076acdcb391bf as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/78061e250519413f8a3076acdcb391bf 2024-12-02T05:17:20,546 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/78061e250519413f8a3076acdcb391bf, entries=150, sequenceid=121, filesize=30.2 K 2024-12-02T05:17:20,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/f88c4bd1e9bc4c42ae18ea13873c4d17 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/f88c4bd1e9bc4c42ae18ea13873c4d17 2024-12-02T05:17:20,551 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/f88c4bd1e9bc4c42ae18ea13873c4d17, entries=150, sequenceid=121, filesize=11.7 K 2024-12-02T05:17:20,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/98941203cae047f1a85718e82a3c7bae as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/98941203cae047f1a85718e82a3c7bae 2024-12-02T05:17:20,559 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/98941203cae047f1a85718e82a3c7bae, entries=150, sequenceid=121, filesize=11.7 K 2024-12-02T05:17:20,560 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ffa3b22adefa33dc88a53908588e6f34 in 875ms, sequenceid=121, compaction requested=true 2024-12-02T05:17:20,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:20,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:20,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-02T05:17:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-02T05:17:20,564 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-02T05:17:20,564 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6890 sec 2024-12-02T05:17:20,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 3.6960 sec 2024-12-02T05:17:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:20,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T05:17:20,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:20,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:20,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:20,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:20,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:20,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:20,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202207bcc37202b4ec9a2209eceb98ea49f_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116640848/Put/seqid=0 2024-12-02T05:17:20,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116700871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:20,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741972_1148 (size=14794) 2024-12-02T05:17:20,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116700873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:20,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116700874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:20,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116700878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:20,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116700918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:20,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116700987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:20,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:20,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116700987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:20,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:20,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116700987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116700987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:20,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:20,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T05:17:21,027 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:21,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-02T05:17:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T05:17:21,030 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:21,030 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:21,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T05:17:21,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T05:17:21,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:21,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:21,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:21,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116701197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:21,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:21,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:21,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116701198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:21,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:21,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:21,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116701198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:21,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116701198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:21,279 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:21,283 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202207bcc37202b4ec9a2209eceb98ea49f_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202207bcc37202b4ec9a2209eceb98ea49f_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:21,284 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/59a79a9496634e01b66aa55d5fcbd6f4, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:21,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/59a79a9496634e01b66aa55d5fcbd6f4 is 175, key is test_row_0/A:col10/1733116640848/Put/seqid=0 2024-12-02T05:17:21,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741973_1149 (size=39749) 2024-12-02T05:17:21,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T05:17:21,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:21,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:21,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:21,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T05:17:21,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T05:17:21,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:21,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:21,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:21,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,511 INFO [master/f49461030c22:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-02T05:17:21,511 INFO [master/f49461030c22:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-02T05:17:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116701507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116701507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116701508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:21,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116701508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:21,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T05:17:21,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:21,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:21,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:21,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T05:17:21,689 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/59a79a9496634e01b66aa55d5fcbd6f4 2024-12-02T05:17:21,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/01cd5f2674064729a97ea072b136891e is 50, key is test_row_0/B:col10/1733116640848/Put/seqid=0 2024-12-02T05:17:21,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741974_1150 (size=12151) 2024-12-02T05:17:21,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/01cd5f2674064729a97ea072b136891e 2024-12-02T05:17:21,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/6c12540c8c0843ffb5162a70d7e175f2 is 50, key is test_row_0/C:col10/1733116640848/Put/seqid=0 2024-12-02T05:17:21,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741975_1151 (size=12151) 2024-12-02T05:17:21,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T05:17:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:21,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:21,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116701927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:21,930 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in ffa3b22adefa33dc88a53908588e6f34 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:21,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:21,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:21,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T05:17:21,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:21,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:21,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:21,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:22,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116702018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:22,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:22,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:22,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:22,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116702018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:22,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116702027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:22,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116702027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:22,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:22,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:22,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T05:17:22,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:22,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:22,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:22,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:22,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:22,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/6c12540c8c0843ffb5162a70d7e175f2 2024-12-02T05:17:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/59a79a9496634e01b66aa55d5fcbd6f4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/59a79a9496634e01b66aa55d5fcbd6f4 2024-12-02T05:17:22,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/59a79a9496634e01b66aa55d5fcbd6f4, entries=200, sequenceid=134, filesize=38.8 K 2024-12-02T05:17:22,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/01cd5f2674064729a97ea072b136891e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/01cd5f2674064729a97ea072b136891e 2024-12-02T05:17:22,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/01cd5f2674064729a97ea072b136891e, entries=150, sequenceid=134, filesize=11.9 K 2024-12-02T05:17:22,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/6c12540c8c0843ffb5162a70d7e175f2 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/6c12540c8c0843ffb5162a70d7e175f2 2024-12-02T05:17:22,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/6c12540c8c0843ffb5162a70d7e175f2, entries=150, sequenceid=134, filesize=11.9 K 2024-12-02T05:17:22,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ffa3b22adefa33dc88a53908588e6f34 in 1302ms, sequenceid=134, compaction requested=true 2024-12-02T05:17:22,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:22,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffa3b22adefa33dc88a53908588e6f34:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:17:22,152 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:17:22,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:22,152 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:17:22,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffa3b22adefa33dc88a53908588e6f34:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:17:22,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:22,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffa3b22adefa33dc88a53908588e6f34:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:17:22,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:22,154 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:17:22,154 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141346 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:17:22,154 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): ffa3b22adefa33dc88a53908588e6f34/A is initiating minor compaction (all files) 2024-12-02T05:17:22,154 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ffa3b22adefa33dc88a53908588e6f34/B is initiating minor compaction (all files) 2024-12-02T05:17:22,154 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffa3b22adefa33dc88a53908588e6f34/A in TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:22,154 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffa3b22adefa33dc88a53908588e6f34/B in TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:22,154 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/597a7acdaea049fb89474c4859324e32, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/1dddeb376ec343afb595c93cc9669663, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/78061e250519413f8a3076acdcb391bf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/59a79a9496634e01b66aa55d5fcbd6f4] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp, totalSize=138.0 K 2024-12-02T05:17:22,154 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/510dff0b1ba246b891b6512f72cb83ac, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/5507035f74594910b8d6a07dde7692fa, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/f88c4bd1e9bc4c42ae18ea13873c4d17, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/01cd5f2674064729a97ea072b136891e] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp, totalSize=47.2 K 2024-12-02T05:17:22,155 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:22,155 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/597a7acdaea049fb89474c4859324e32, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/1dddeb376ec343afb595c93cc9669663, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/78061e250519413f8a3076acdcb391bf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/59a79a9496634e01b66aa55d5fcbd6f4] 2024-12-02T05:17:22,155 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 510dff0b1ba246b891b6512f72cb83ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733116636315 2024-12-02T05:17:22,155 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 597a7acdaea049fb89474c4859324e32, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733116636315 2024-12-02T05:17:22,156 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dddeb376ec343afb595c93cc9669663, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733116636670 2024-12-02T05:17:22,156 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 5507035f74594910b8d6a07dde7692fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733116636670 2024-12-02T05:17:22,158 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting f88c4bd1e9bc4c42ae18ea13873c4d17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733116638568 2024-12-02T05:17:22,158 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78061e250519413f8a3076acdcb391bf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733116638568 2024-12-02T05:17:22,159 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 01cd5f2674064729a97ea072b136891e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733116639764 2024-12-02T05:17:22,159 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59a79a9496634e01b66aa55d5fcbd6f4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733116639764 2024-12-02T05:17:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T05:17:22,174 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffa3b22adefa33dc88a53908588e6f34#B#compaction#128 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:22,175 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/a1a0d61b930240169b0081732b1056c3 is 50, key is test_row_0/B:col10/1733116640848/Put/seqid=0 2024-12-02T05:17:22,185 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:22,193 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202b51ff18a338b4a4b95be2841a5a1aa5e_ffa3b22adefa33dc88a53908588e6f34 store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:22,196 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202b51ff18a338b4a4b95be2841a5a1aa5e_ffa3b22adefa33dc88a53908588e6f34, store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:22,197 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b51ff18a338b4a4b95be2841a5a1aa5e_ffa3b22adefa33dc88a53908588e6f34 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:22,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741976_1152 (size=12425) 2024-12-02T05:17:22,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741977_1153 (size=4469) 2024-12-02T05:17:22,223 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffa3b22adefa33dc88a53908588e6f34#A#compaction#129 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:22,223 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/85253287918f436c97a7c2831db250bd is 175, key is test_row_0/A:col10/1733116640848/Put/seqid=0 2024-12-02T05:17:22,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741978_1154 (size=31379) 2024-12-02T05:17:22,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T05:17:22,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:22,253 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T05:17:22,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:22,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:22,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:22,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:22,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:22,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:22,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c91ee6e3b08046c5bd8641eb4eda2566_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116640871/Put/seqid=0 2024-12-02T05:17:22,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741979_1155 (size=12304) 2024-12-02T05:17:22,611 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/a1a0d61b930240169b0081732b1056c3 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/a1a0d61b930240169b0081732b1056c3 2024-12-02T05:17:22,616 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffa3b22adefa33dc88a53908588e6f34/B of ffa3b22adefa33dc88a53908588e6f34 into a1a0d61b930240169b0081732b1056c3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:22,616 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:22,616 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., storeName=ffa3b22adefa33dc88a53908588e6f34/B, priority=12, startTime=1733116642152; duration=0sec 2024-12-02T05:17:22,616 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:22,616 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffa3b22adefa33dc88a53908588e6f34:B 2024-12-02T05:17:22,616 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:17:22,618 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:17:22,618 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ffa3b22adefa33dc88a53908588e6f34/C is initiating minor compaction (all files) 2024-12-02T05:17:22,618 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffa3b22adefa33dc88a53908588e6f34/C in TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:22,618 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/b9799e649e034706bf6e64d161363b6a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/d012ef187ca14b2ab67029227165f11f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/98941203cae047f1a85718e82a3c7bae, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/6c12540c8c0843ffb5162a70d7e175f2] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp, totalSize=47.2 K 2024-12-02T05:17:22,619 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting b9799e649e034706bf6e64d161363b6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733116636315 2024-12-02T05:17:22,619 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting d012ef187ca14b2ab67029227165f11f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733116636670 2024-12-02T05:17:22,619 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 98941203cae047f1a85718e82a3c7bae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733116638568 2024-12-02T05:17:22,619 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c12540c8c0843ffb5162a70d7e175f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733116639764 2024-12-02T05:17:22,628 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffa3b22adefa33dc88a53908588e6f34#C#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:22,629 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/3b8824b1016c4f48a23fb5219fc08a3b is 50, key is test_row_0/C:col10/1733116640848/Put/seqid=0 2024-12-02T05:17:22,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741980_1156 (size=12425) 2024-12-02T05:17:22,636 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/85253287918f436c97a7c2831db250bd as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/85253287918f436c97a7c2831db250bd 2024-12-02T05:17:22,641 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/3b8824b1016c4f48a23fb5219fc08a3b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/3b8824b1016c4f48a23fb5219fc08a3b 2024-12-02T05:17:22,644 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffa3b22adefa33dc88a53908588e6f34/A of ffa3b22adefa33dc88a53908588e6f34 into 85253287918f436c97a7c2831db250bd(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:22,644 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:22,644 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., storeName=ffa3b22adefa33dc88a53908588e6f34/A, priority=12, startTime=1733116642152; duration=0sec 2024-12-02T05:17:22,645 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:22,645 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffa3b22adefa33dc88a53908588e6f34:A 2024-12-02T05:17:22,650 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffa3b22adefa33dc88a53908588e6f34/C of ffa3b22adefa33dc88a53908588e6f34 into 3b8824b1016c4f48a23fb5219fc08a3b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:22,651 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:22,651 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., storeName=ffa3b22adefa33dc88a53908588e6f34/C, priority=12, startTime=1733116642152; duration=0sec 2024-12-02T05:17:22,651 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:22,651 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffa3b22adefa33dc88a53908588e6f34:C 2024-12-02T05:17:22,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:22,668 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c91ee6e3b08046c5bd8641eb4eda2566_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c91ee6e3b08046c5bd8641eb4eda2566_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/c79ee38ce3b9400ab218bff7de31dffc, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/c79ee38ce3b9400ab218bff7de31dffc is 175, key is test_row_0/A:col10/1733116640871/Put/seqid=0 2024-12-02T05:17:22,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741981_1157 (size=31105) 2024-12-02T05:17:22,677 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/c79ee38ce3b9400ab218bff7de31dffc 2024-12-02T05:17:22,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/771c2486e3c748af8db36839163c2337 is 50, key is test_row_0/B:col10/1733116640871/Put/seqid=0 2024-12-02T05:17:22,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741982_1158 (size=12151) 2024-12-02T05:17:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:23,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:23,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116703047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116703047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116703047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116703048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,055 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in ffa3b22adefa33dc88a53908588e6f34 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,096 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/771c2486e3c748af8db36839163c2337 2024-12-02T05:17:23,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/dde2008cf7f6450cbefb9bb52057e524 is 50, key is test_row_0/C:col10/1733116640871/Put/seqid=0 2024-12-02T05:17:23,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741983_1159 (size=12151) 2024-12-02T05:17:23,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116703157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116703158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116703168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T05:17:23,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116703368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116703368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116703377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,523 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/dde2008cf7f6450cbefb9bb52057e524 2024-12-02T05:17:23,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/c79ee38ce3b9400ab218bff7de31dffc as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/c79ee38ce3b9400ab218bff7de31dffc 2024-12-02T05:17:23,532 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/c79ee38ce3b9400ab218bff7de31dffc, entries=150, sequenceid=153, filesize=30.4 K 2024-12-02T05:17:23,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/771c2486e3c748af8db36839163c2337 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/771c2486e3c748af8db36839163c2337 2024-12-02T05:17:23,539 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/771c2486e3c748af8db36839163c2337, entries=150, sequenceid=153, filesize=11.9 K 2024-12-02T05:17:23,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/dde2008cf7f6450cbefb9bb52057e524 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/dde2008cf7f6450cbefb9bb52057e524 2024-12-02T05:17:23,545 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/dde2008cf7f6450cbefb9bb52057e524, entries=150, sequenceid=153, filesize=11.9 K 2024-12-02T05:17:23,546 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ffa3b22adefa33dc88a53908588e6f34 in 1292ms, sequenceid=153, compaction requested=false 2024-12-02T05:17:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-02T05:17:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-02T05:17:23,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-02T05:17:23,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5170 sec 2024-12-02T05:17:23,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.5210 sec 2024-12-02T05:17:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:23,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T05:17:23,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:23,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:23,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:23,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:23,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:23,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:23,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026ac4fc75d55c4aacb6bda5300eb46861_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116643045/Put/seqid=0 2024-12-02T05:17:23,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741984_1160 (size=14794) 2024-12-02T05:17:23,692 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:23,695 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026ac4fc75d55c4aacb6bda5300eb46861_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026ac4fc75d55c4aacb6bda5300eb46861_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:23,696 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/79b90d0344854d149f58cd9054b80582, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:23,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/79b90d0344854d149f58cd9054b80582 is 175, key is test_row_0/A:col10/1733116643045/Put/seqid=0 2024-12-02T05:17:23,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741985_1161 (size=39749) 2024-12-02T05:17:23,703 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/79b90d0344854d149f58cd9054b80582 2024-12-02T05:17:23,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116703704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116703705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116703705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/af0f10f2e5db464a8956493fa6fe534a is 50, key is test_row_0/B:col10/1733116643045/Put/seqid=0 2024-12-02T05:17:23,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741986_1162 (size=12151) 2024-12-02T05:17:23,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/af0f10f2e5db464a8956493fa6fe534a 2024-12-02T05:17:23,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/c94d3e1e92824421bc3c320344d681e4 is 50, key is test_row_0/C:col10/1733116643045/Put/seqid=0 2024-12-02T05:17:23,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741987_1163 (size=12151) 2024-12-02T05:17:23,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116703807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116703818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116703818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:23,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116703947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:23,950 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in ffa3b22adefa33dc88a53908588e6f34 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:24,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116704017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:24,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116704028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:24,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116704028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:24,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/c94d3e1e92824421bc3c320344d681e4 2024-12-02T05:17:24,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/79b90d0344854d149f58cd9054b80582 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/79b90d0344854d149f58cd9054b80582 2024-12-02T05:17:24,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/79b90d0344854d149f58cd9054b80582, entries=200, sequenceid=170, filesize=38.8 K 2024-12-02T05:17:24,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/af0f10f2e5db464a8956493fa6fe534a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/af0f10f2e5db464a8956493fa6fe534a 2024-12-02T05:17:24,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116704327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:24,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/af0f10f2e5db464a8956493fa6fe534a, entries=150, sequenceid=170, filesize=11.9 K 2024-12-02T05:17:24,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/c94d3e1e92824421bc3c320344d681e4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/c94d3e1e92824421bc3c320344d681e4 2024-12-02T05:17:24,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/c94d3e1e92824421bc3c320344d681e4, entries=150, sequenceid=170, filesize=11.9 K 2024-12-02T05:17:24,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ffa3b22adefa33dc88a53908588e6f34 in 661ms, sequenceid=170, compaction requested=true 2024-12-02T05:17:24,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:24,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffa3b22adefa33dc88a53908588e6f34:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:17:24,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:24,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffa3b22adefa33dc88a53908588e6f34:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:17:24,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:24,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffa3b22adefa33dc88a53908588e6f34:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:17:24,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:24,340 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:24,341 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:24,342 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:24,342 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ffa3b22adefa33dc88a53908588e6f34/B is initiating minor compaction (all files) 2024-12-02T05:17:24,343 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffa3b22adefa33dc88a53908588e6f34/B in TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:24,343 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/a1a0d61b930240169b0081732b1056c3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/771c2486e3c748af8db36839163c2337, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/af0f10f2e5db464a8956493fa6fe534a] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp, totalSize=35.9 K 2024-12-02T05:17:24,343 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting a1a0d61b930240169b0081732b1056c3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733116639764 2024-12-02T05:17:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:24,344 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:24,344 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): ffa3b22adefa33dc88a53908588e6f34/A is initiating minor compaction (all files) 2024-12-02T05:17:24,344 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffa3b22adefa33dc88a53908588e6f34/A in TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:24,344 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/85253287918f436c97a7c2831db250bd, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/c79ee38ce3b9400ab218bff7de31dffc, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/79b90d0344854d149f58cd9054b80582] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp, totalSize=99.8 K 2024-12-02T05:17:24,344 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 771c2486e3c748af8db36839163c2337, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733116640871 2024-12-02T05:17:24,345 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting af0f10f2e5db464a8956493fa6fe534a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733116643045 2024-12-02T05:17:24,345 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:24,345 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/85253287918f436c97a7c2831db250bd, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/c79ee38ce3b9400ab218bff7de31dffc, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/79b90d0344854d149f58cd9054b80582] 2024-12-02T05:17:24,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-12-02T05:17:24,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:24,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:24,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:24,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116643701, max timestamp=1733116644345Num uniques 859; 2024-12-02T05:17:24,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:24,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:24,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:17:24,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:24,349 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85253287918f436c97a7c2831db250bd, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733116639764 2024-12-02T05:17:24,349 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting c79ee38ce3b9400ab218bff7de31dffc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733116640871 2024-12-02T05:17:24,350 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79b90d0344854d149f58cd9054b80582, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733116643045 2024-12-02T05:17:24,382 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffa3b22adefa33dc88a53908588e6f34#B#compaction#137 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:24,383 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/582b6a6c69d5408ead1c1f1c12928129 is 50, key is test_row_0/B:col10/1733116643045/Put/seqid=0 2024-12-02T05:17:24,385 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:24,388 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412023ffe2642b1dc4bdea0367f78c9df0372_ffa3b22adefa33dc88a53908588e6f34 store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:24,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202692d98bc2d9449b38879979e7bb4e4d8_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116643704/Put/seqid=0 2024-12-02T05:17:24,393 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412023ffe2642b1dc4bdea0367f78c9df0372_ffa3b22adefa33dc88a53908588e6f34, store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:24,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:24,394 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412023ffe2642b1dc4bdea0367f78c9df0372_ffa3b22adefa33dc88a53908588e6f34 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116704386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:24,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116704388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:24,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:24,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741988_1164 (size=17284) 2024-12-02T05:17:24,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741989_1165 (size=12527) 2024-12-02T05:17:24,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741990_1166 (size=4469) 2024-12-02T05:17:24,416 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffa3b22adefa33dc88a53908588e6f34#A#compaction#138 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:24,417 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/dbba756c3a1d476f9405b4a7a52e695f is 175, key is test_row_0/A:col10/1733116643045/Put/seqid=0 2024-12-02T05:17:24,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741991_1167 (size=31481) 2024-12-02T05:17:24,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116704497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:24,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116704497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:24,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116704708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:24,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116704708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:24,812 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:24,815 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202692d98bc2d9449b38879979e7bb4e4d8_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202692d98bc2d9449b38879979e7bb4e4d8_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:24,816 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/c7172f09929a445494e838906cd6a031, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:24,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/c7172f09929a445494e838906cd6a031 is 175, key is test_row_0/A:col10/1733116643704/Put/seqid=0 2024-12-02T05:17:24,817 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/582b6a6c69d5408ead1c1f1c12928129 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/582b6a6c69d5408ead1c1f1c12928129 2024-12-02T05:17:24,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741992_1168 (size=48389) 2024-12-02T05:17:24,822 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/c7172f09929a445494e838906cd6a031 2024-12-02T05:17:24,822 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffa3b22adefa33dc88a53908588e6f34/B of ffa3b22adefa33dc88a53908588e6f34 into 582b6a6c69d5408ead1c1f1c12928129(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:24,822 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:24,822 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., storeName=ffa3b22adefa33dc88a53908588e6f34/B, priority=13, startTime=1733116644340; duration=0sec 2024-12-02T05:17:24,822 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:24,822 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffa3b22adefa33dc88a53908588e6f34:B 2024-12-02T05:17:24,822 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:24,825 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:24,825 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ffa3b22adefa33dc88a53908588e6f34/C is initiating minor compaction (all files) 2024-12-02T05:17:24,825 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffa3b22adefa33dc88a53908588e6f34/C in TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:24,825 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/3b8824b1016c4f48a23fb5219fc08a3b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/dde2008cf7f6450cbefb9bb52057e524, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/c94d3e1e92824421bc3c320344d681e4] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp, totalSize=35.9 K 2024-12-02T05:17:24,825 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b8824b1016c4f48a23fb5219fc08a3b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733116639764 2024-12-02T05:17:24,826 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting dde2008cf7f6450cbefb9bb52057e524, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733116640871 2024-12-02T05:17:24,827 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting c94d3e1e92824421bc3c320344d681e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733116643045 2024-12-02T05:17:24,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/d8a7c0be78bb4089ad021a4394abe1de is 50, key is test_row_0/B:col10/1733116643704/Put/seqid=0 2024-12-02T05:17:24,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741993_1169 (size=12151) 2024-12-02T05:17:24,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:24,838 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffa3b22adefa33dc88a53908588e6f34#C#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116704838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:24,839 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/cfe17072c8d949d6b98a27f70e46e314 is 50, key is test_row_0/C:col10/1733116643045/Put/seqid=0 2024-12-02T05:17:24,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:24,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:24,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741994_1170 (size=12527) 2024-12-02T05:17:24,851 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/dbba756c3a1d476f9405b4a7a52e695f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/dbba756c3a1d476f9405b4a7a52e695f 2024-12-02T05:17:24,857 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/cfe17072c8d949d6b98a27f70e46e314 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/cfe17072c8d949d6b98a27f70e46e314 2024-12-02T05:17:24,858 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffa3b22adefa33dc88a53908588e6f34/A of ffa3b22adefa33dc88a53908588e6f34 into dbba756c3a1d476f9405b4a7a52e695f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:24,858 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:24,858 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., storeName=ffa3b22adefa33dc88a53908588e6f34/A, priority=13, startTime=1733116644340; duration=0sec 2024-12-02T05:17:24,858 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:24,858 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffa3b22adefa33dc88a53908588e6f34:A 2024-12-02T05:17:24,865 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffa3b22adefa33dc88a53908588e6f34/C of ffa3b22adefa33dc88a53908588e6f34 into cfe17072c8d949d6b98a27f70e46e314(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:24,865 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:24,865 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., storeName=ffa3b22adefa33dc88a53908588e6f34/C, priority=13, startTime=1733116644340; duration=0sec 2024-12-02T05:17:24,865 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:24,865 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffa3b22adefa33dc88a53908588e6f34:C 2024-12-02T05:17:25,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116705018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:25,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116705018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:25,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116705067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:25,070 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in ffa3b22adefa33dc88a53908588e6f34 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T05:17:25,198 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:25,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-02T05:17:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T05:17:25,200 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:25,200 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:25,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:25,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/d8a7c0be78bb4089ad021a4394abe1de 2024-12-02T05:17:25,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/100cc090bad1421fa34ac472d1187d79 is 50, key is test_row_0/C:col10/1733116643704/Put/seqid=0 2024-12-02T05:17:25,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741995_1171 (size=12151) 2024-12-02T05:17:25,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/100cc090bad1421fa34ac472d1187d79 2024-12-02T05:17:25,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/c7172f09929a445494e838906cd6a031 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/c7172f09929a445494e838906cd6a031 2024-12-02T05:17:25,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/c7172f09929a445494e838906cd6a031, entries=250, sequenceid=192, filesize=47.3 K 2024-12-02T05:17:25,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/d8a7c0be78bb4089ad021a4394abe1de as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/d8a7c0be78bb4089ad021a4394abe1de 2024-12-02T05:17:25,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/d8a7c0be78bb4089ad021a4394abe1de, entries=150, sequenceid=192, filesize=11.9 K 2024-12-02T05:17:25,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/100cc090bad1421fa34ac472d1187d79 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/100cc090bad1421fa34ac472d1187d79 2024-12-02T05:17:25,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/100cc090bad1421fa34ac472d1187d79, entries=150, sequenceid=192, filesize=11.9 K 2024-12-02T05:17:25,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=80.51 KB/82440 for ffa3b22adefa33dc88a53908588e6f34 in 956ms, sequenceid=192, compaction requested=false 2024-12-02T05:17:25,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T05:17:25,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T05:17:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:25,353 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T05:17:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:25,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120264c77214d48f4df19ab1f3f2eafb3cad_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116644386/Put/seqid=0 2024-12-02T05:17:25,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741996_1172 (size=12304) 2024-12-02T05:17:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T05:17:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:25,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:25,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116705554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:25,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116705558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:25,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116705667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:25,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116705667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:25,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:25,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:25,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:25,773 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120264c77214d48f4df19ab1f3f2eafb3cad_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120264c77214d48f4df19ab1f3f2eafb3cad_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:25,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/7174f37b5c494e728075c11acd0590f7, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:25,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/7174f37b5c494e728075c11acd0590f7 is 175, key is test_row_0/A:col10/1733116644386/Put/seqid=0 2024-12-02T05:17:25,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741997_1173 (size=31105) 2024-12-02T05:17:25,782 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/7174f37b5c494e728075c11acd0590f7 2024-12-02T05:17:25,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/2ac898353f7a403db65204bfb9fbedbc is 50, key is test_row_0/B:col10/1733116644386/Put/seqid=0 2024-12-02T05:17:25,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741998_1174 (size=12151) 2024-12-02T05:17:25,807 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/2ac898353f7a403db65204bfb9fbedbc 2024-12-02T05:17:25,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/963377caf10a400e870d2feddd57d5fb is 50, key is test_row_0/C:col10/1733116644386/Put/seqid=0 2024-12-02T05:17:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T05:17:25,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741999_1175 (size=12151) 2024-12-02T05:17:25,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116705847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:25,850 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_0' in ffa3b22adefa33dc88a53908588e6f34 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2146 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:25,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:25,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116705877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:25,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:25,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116705877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:26,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116706187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:26,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:26,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:26,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:26,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116706187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:26,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:26,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:26,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:26,236 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/963377caf10a400e870d2feddd57d5fb 2024-12-02T05:17:26,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/7174f37b5c494e728075c11acd0590f7 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/7174f37b5c494e728075c11acd0590f7 2024-12-02T05:17:26,245 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/7174f37b5c494e728075c11acd0590f7, entries=150, sequenceid=210, filesize=30.4 K 2024-12-02T05:17:26,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/2ac898353f7a403db65204bfb9fbedbc as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/2ac898353f7a403db65204bfb9fbedbc 2024-12-02T05:17:26,250 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/2ac898353f7a403db65204bfb9fbedbc, entries=150, sequenceid=210, filesize=11.9 K 2024-12-02T05:17:26,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/963377caf10a400e870d2feddd57d5fb as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/963377caf10a400e870d2feddd57d5fb 2024-12-02T05:17:26,255 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/963377caf10a400e870d2feddd57d5fb, entries=150, sequenceid=210, filesize=11.9 K 2024-12-02T05:17:26,256 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ffa3b22adefa33dc88a53908588e6f34 in 903ms, sequenceid=210, compaction requested=true 2024-12-02T05:17:26,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:26,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:26,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-02T05:17:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-02T05:17:26,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-02T05:17:26,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0580 sec 2024-12-02T05:17:26,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.0620 sec 2024-12-02T05:17:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T05:17:26,338 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:26,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-02T05:17:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T05:17:26,340 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:26,340 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:26,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T05:17:26,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-02T05:17:26,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:26,493 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T05:17:26,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:26,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:26,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:26,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:26,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:26,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:26,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024366ad00188e4e88a2e5f58711993912_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116645549/Put/seqid=0 2024-12-02T05:17:26,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742000_1176 (size=12304) 2024-12-02T05:17:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T05:17:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:26,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:26,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116706719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:26,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116706720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:26,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:26,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:26,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:26,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:26,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:26,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116706828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:26,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:26,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:26,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116706828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:26,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:26,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:26,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:26,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:26,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:26,913 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024366ad00188e4e88a2e5f58711993912_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024366ad00188e4e88a2e5f58711993912_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:26,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/af985620a1b74113a80aecb96d463319, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:26,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/af985620a1b74113a80aecb96d463319 is 175, key is test_row_0/A:col10/1733116645549/Put/seqid=0 2024-12-02T05:17:26,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742001_1177 (size=31105) 2024-12-02T05:17:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T05:17:27,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116707037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:27,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:27,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116707037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:27,328 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/af985620a1b74113a80aecb96d463319 2024-12-02T05:17:27,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/299301c6762f4221befde06feefe0641 is 50, key is test_row_0/B:col10/1733116645549/Put/seqid=0 2024-12-02T05:17:27,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742002_1178 (size=12151) 2024-12-02T05:17:27,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:27,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116707348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116707348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:27,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:27,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T05:17:27,746 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/299301c6762f4221befde06feefe0641 2024-12-02T05:17:27,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/e519579efa86469ea4239e7017f86607 is 50, key is test_row_0/C:col10/1733116645549/Put/seqid=0 2024-12-02T05:17:27,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742003_1179 (size=12151) 2024-12-02T05:17:27,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116707858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:27,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116707858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:27,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116707867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:27,872 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_0' in ffa3b22adefa33dc88a53908588e6f34 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:27,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49828 deadline: 1733116707967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:27,971 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in ffa3b22adefa33dc88a53908588e6f34 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:27,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:27,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:28,159 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/e519579efa86469ea4239e7017f86607 2024-12-02T05:17:28,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/af985620a1b74113a80aecb96d463319 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/af985620a1b74113a80aecb96d463319 2024-12-02T05:17:28,169 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/af985620a1b74113a80aecb96d463319, entries=150, sequenceid=227, filesize=30.4 K 2024-12-02T05:17:28,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/299301c6762f4221befde06feefe0641 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/299301c6762f4221befde06feefe0641 2024-12-02T05:17:28,175 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/299301c6762f4221befde06feefe0641, entries=150, sequenceid=227, filesize=11.9 K 2024-12-02T05:17:28,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/e519579efa86469ea4239e7017f86607 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/e519579efa86469ea4239e7017f86607 2024-12-02T05:17:28,180 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/e519579efa86469ea4239e7017f86607, entries=150, sequenceid=227, filesize=11.9 K 2024-12-02T05:17:28,181 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ffa3b22adefa33dc88a53908588e6f34 in 1688ms, sequenceid=227, compaction requested=true 2024-12-02T05:17:28,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:28,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:28,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-02T05:17:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-02T05:17:28,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-02T05:17:28,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8430 sec 2024-12-02T05:17:28,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.8470 sec 2024-12-02T05:17:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T05:17:28,488 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:28,488 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-02T05:17:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T05:17:28,490 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:28,491 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:28,491 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T05:17:28,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T05:17:28,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:28,643 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T05:17:28,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:28,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:28,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:28,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:28,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:28,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:28,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022aa6256252da412a80706b92e5510700_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116646719/Put/seqid=0 2024-12-02T05:17:28,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742004_1180 (size=12304) 2024-12-02T05:17:28,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:28,665 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022aa6256252da412a80706b92e5510700_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022aa6256252da412a80706b92e5510700_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:28,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/f3213610b43d4fde98ddf0e2b64f6177, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:28,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/f3213610b43d4fde98ddf0e2b64f6177 is 175, key is test_row_0/A:col10/1733116646719/Put/seqid=0 2024-12-02T05:17:28,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742005_1181 (size=31105) 2024-12-02T05:17:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T05:17:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:28,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:28,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116708892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:28,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116708892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:29,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116708998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:29,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116708998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:29,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1733116709088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:29,089 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in ffa3b22adefa33dc88a53908588e6f34 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:29,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:29,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:29,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T05:17:29,194 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/f3213610b43d4fde98ddf0e2b64f6177 2024-12-02T05:17:29,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/3cc432cbecaa4f14b91aa095f32dfed4 is 50, key is test_row_0/B:col10/1733116646719/Put/seqid=0 2024-12-02T05:17:29,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742006_1182 (size=12151) 2024-12-02T05:17:29,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116709208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:29,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116709208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:29,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116709517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:29,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116709517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:29,606 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/3cc432cbecaa4f14b91aa095f32dfed4 2024-12-02T05:17:29,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/9bbc10620d3a480584803b38c90ca9fb is 50, key is test_row_0/C:col10/1733116646719/Put/seqid=0 2024-12-02T05:17:29,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742007_1183 (size=12151) 2024-12-02T05:17:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T05:17:30,020 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/9bbc10620d3a480584803b38c90ca9fb 2024-12-02T05:17:30,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/f3213610b43d4fde98ddf0e2b64f6177 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/f3213610b43d4fde98ddf0e2b64f6177 2024-12-02T05:17:30,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:30,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116710027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116710027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:30,036 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/f3213610b43d4fde98ddf0e2b64f6177, entries=150, sequenceid=242, filesize=30.4 K 2024-12-02T05:17:30,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/3cc432cbecaa4f14b91aa095f32dfed4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/3cc432cbecaa4f14b91aa095f32dfed4 2024-12-02T05:17:30,042 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/3cc432cbecaa4f14b91aa095f32dfed4, entries=150, sequenceid=242, filesize=11.9 K 2024-12-02T05:17:30,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/9bbc10620d3a480584803b38c90ca9fb as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/9bbc10620d3a480584803b38c90ca9fb 2024-12-02T05:17:30,048 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/9bbc10620d3a480584803b38c90ca9fb, entries=150, sequenceid=242, filesize=11.9 K 2024-12-02T05:17:30,049 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ffa3b22adefa33dc88a53908588e6f34 in 1406ms, sequenceid=242, compaction requested=true 2024-12-02T05:17:30,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:30,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:30,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-02T05:17:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-02T05:17:30,053 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-02T05:17:30,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5600 sec 2024-12-02T05:17:30,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5660 sec 2024-12-02T05:17:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T05:17:30,638 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:30,638 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-02T05:17:30,640 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T05:17:30,641 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:30,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T05:17:30,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T05:17:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:30,794 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T05:17:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:30,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202dec27cf89be741498dae61493a9b5bdc_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116648886/Put/seqid=0 2024-12-02T05:17:30,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742008_1184 (size=12354) 2024-12-02T05:17:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T05:17:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:31,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:31,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116711066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:31,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116711071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:31,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:31,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116711178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:31,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116711178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:31,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:31,226 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202dec27cf89be741498dae61493a9b5bdc_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dec27cf89be741498dae61493a9b5bdc_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:31,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/3a334263204840b7b1b8c506400b9582, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:31,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/3a334263204840b7b1b8c506400b9582 is 175, key is test_row_0/A:col10/1733116648886/Put/seqid=0 2024-12-02T05:17:31,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742009_1185 (size=31155) 2024-12-02T05:17:31,233 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/3a334263204840b7b1b8c506400b9582 2024-12-02T05:17:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/fc5683f7930a4cf1a0470f151b32bf1b is 50, key is test_row_0/B:col10/1733116648886/Put/seqid=0 2024-12-02T05:17:31,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742010_1186 (size=12201) 2024-12-02T05:17:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T05:17:31,278 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T05:17:31,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:31,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116711387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116711387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:31,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:31,648 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/fc5683f7930a4cf1a0470f151b32bf1b 2024-12-02T05:17:31,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/8cc1e86f6581445a941b0020d28dbf9c is 50, key is test_row_0/C:col10/1733116648886/Put/seqid=0 2024-12-02T05:17:31,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742011_1187 (size=12201) 2024-12-02T05:17:31,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116711698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:31,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116711698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T05:17:31,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49844 deadline: 1733116711897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:31,908 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_0' in ffa3b22adefa33dc88a53908588e6f34 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:32,061 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/8cc1e86f6581445a941b0020d28dbf9c 2024-12-02T05:17:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/3a334263204840b7b1b8c506400b9582 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/3a334263204840b7b1b8c506400b9582 2024-12-02T05:17:32,070 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/3a334263204840b7b1b8c506400b9582, entries=150, sequenceid=259, filesize=30.4 K 2024-12-02T05:17:32,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/fc5683f7930a4cf1a0470f151b32bf1b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/fc5683f7930a4cf1a0470f151b32bf1b 2024-12-02T05:17:32,075 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/fc5683f7930a4cf1a0470f151b32bf1b, entries=150, sequenceid=259, filesize=11.9 K 2024-12-02T05:17:32,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/8cc1e86f6581445a941b0020d28dbf9c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/8cc1e86f6581445a941b0020d28dbf9c 2024-12-02T05:17:32,080 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/8cc1e86f6581445a941b0020d28dbf9c, entries=150, sequenceid=259, filesize=11.9 K 2024-12-02T05:17:32,081 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ffa3b22adefa33dc88a53908588e6f34 in 1287ms, sequenceid=259, compaction requested=true 2024-12-02T05:17:32,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:32,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:32,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-02T05:17:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-02T05:17:32,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-02T05:17:32,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4420 sec 2024-12-02T05:17:32,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4470 sec 2024-12-02T05:17:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:32,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T05:17:32,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:32,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:32,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:32,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:32,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:32,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:32,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b160eabde3474092923e841021cb4319_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116652212/Put/seqid=0 2024-12-02T05:17:32,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742012_1188 (size=17534) 2024-12-02T05:17:32,224 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:32,228 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b160eabde3474092923e841021cb4319_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b160eabde3474092923e841021cb4319_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:32,229 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/8bde1e7ca1364c2ab53638304e78c2c5, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:32,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/8bde1e7ca1364c2ab53638304e78c2c5 is 175, key is test_row_0/A:col10/1733116652212/Put/seqid=0 2024-12-02T05:17:32,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742013_1189 (size=48639) 2024-12-02T05:17:32,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116712245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:32,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:32,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116712245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:32,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:32,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116712357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:32,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116712357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:32,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116712568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:32,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116712568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:32,634 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/8bde1e7ca1364c2ab53638304e78c2c5 2024-12-02T05:17:32,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/4ee70e62afa94892b85f027c78bb6600 is 50, key is test_row_0/B:col10/1733116652212/Put/seqid=0 2024-12-02T05:17:32,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742014_1190 (size=12301) 2024-12-02T05:17:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T05:17:32,788 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:32,788 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-02T05:17:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T05:17:32,790 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:32,791 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:32,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:32,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116712877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116712877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T05:17:32,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T05:17:32,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:32,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:32,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:32,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:32,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:33,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/4ee70e62afa94892b85f027c78bb6600 2024-12-02T05:17:33,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/fa9dfbf351494378a823ebf899ecf1e0 is 50, key is test_row_0/C:col10/1733116652212/Put/seqid=0 2024-12-02T05:17:33,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742015_1191 (size=12301) 2024-12-02T05:17:33,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/fa9dfbf351494378a823ebf899ecf1e0 2024-12-02T05:17:33,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/8bde1e7ca1364c2ab53638304e78c2c5 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/8bde1e7ca1364c2ab53638304e78c2c5 2024-12-02T05:17:33,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/8bde1e7ca1364c2ab53638304e78c2c5, entries=250, sequenceid=276, filesize=47.5 K 2024-12-02T05:17:33,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/4ee70e62afa94892b85f027c78bb6600 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/4ee70e62afa94892b85f027c78bb6600 2024-12-02T05:17:33,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/4ee70e62afa94892b85f027c78bb6600, entries=150, sequenceid=276, filesize=12.0 K 2024-12-02T05:17:33,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/fa9dfbf351494378a823ebf899ecf1e0 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/fa9dfbf351494378a823ebf899ecf1e0 2024-12-02T05:17:33,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/fa9dfbf351494378a823ebf899ecf1e0, entries=150, sequenceid=276, filesize=12.0 K 2024-12-02T05:17:33,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ffa3b22adefa33dc88a53908588e6f34 in 871ms, sequenceid=276, compaction requested=true 2024-12-02T05:17:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffa3b22adefa33dc88a53908588e6f34:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:17:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:33,083 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-12-02T05:17:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffa3b22adefa33dc88a53908588e6f34:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:17:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:33,083 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-12-02T05:17:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffa3b22adefa33dc88a53908588e6f34:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:17:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:33,085 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 252979 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-12-02T05:17:33,085 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 85633 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-12-02T05:17:33,085 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ffa3b22adefa33dc88a53908588e6f34/A is initiating minor compaction (all files) 2024-12-02T05:17:33,085 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): ffa3b22adefa33dc88a53908588e6f34/B is initiating minor compaction (all files) 2024-12-02T05:17:33,085 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffa3b22adefa33dc88a53908588e6f34/A in TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:33,085 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffa3b22adefa33dc88a53908588e6f34/B in TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:33,085 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/dbba756c3a1d476f9405b4a7a52e695f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/c7172f09929a445494e838906cd6a031, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/7174f37b5c494e728075c11acd0590f7, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/af985620a1b74113a80aecb96d463319, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/f3213610b43d4fde98ddf0e2b64f6177, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/3a334263204840b7b1b8c506400b9582, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/8bde1e7ca1364c2ab53638304e78c2c5] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp, totalSize=247.0 K 2024-12-02T05:17:33,085 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/582b6a6c69d5408ead1c1f1c12928129, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/d8a7c0be78bb4089ad021a4394abe1de, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/2ac898353f7a403db65204bfb9fbedbc, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/299301c6762f4221befde06feefe0641, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/3cc432cbecaa4f14b91aa095f32dfed4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/fc5683f7930a4cf1a0470f151b32bf1b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/4ee70e62afa94892b85f027c78bb6600] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp, totalSize=83.6 K 2024-12-02T05:17:33,085 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:33,085 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/dbba756c3a1d476f9405b4a7a52e695f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/c7172f09929a445494e838906cd6a031, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/7174f37b5c494e728075c11acd0590f7, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/af985620a1b74113a80aecb96d463319, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/f3213610b43d4fde98ddf0e2b64f6177, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/3a334263204840b7b1b8c506400b9582, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/8bde1e7ca1364c2ab53638304e78c2c5] 2024-12-02T05:17:33,085 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 582b6a6c69d5408ead1c1f1c12928129, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733116643045 2024-12-02T05:17:33,085 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting dbba756c3a1d476f9405b4a7a52e695f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733116643045 2024-12-02T05:17:33,086 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8a7c0be78bb4089ad021a4394abe1de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733116643701 2024-12-02T05:17:33,086 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting c7172f09929a445494e838906cd6a031, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733116643701 2024-12-02T05:17:33,086 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ac898353f7a403db65204bfb9fbedbc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733116644375 2024-12-02T05:17:33,086 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 7174f37b5c494e728075c11acd0590f7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733116644375 2024-12-02T05:17:33,086 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 299301c6762f4221befde06feefe0641, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733116645530 2024-12-02T05:17:33,086 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting af985620a1b74113a80aecb96d463319, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733116645530 2024-12-02T05:17:33,087 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cc432cbecaa4f14b91aa095f32dfed4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733116646714 2024-12-02T05:17:33,087 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting f3213610b43d4fde98ddf0e2b64f6177, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733116646714 2024-12-02T05:17:33,087 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc5683f7930a4cf1a0470f151b32bf1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733116648886 2024-12-02T05:17:33,087 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a334263204840b7b1b8c506400b9582, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733116648886 2024-12-02T05:17:33,087 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ee70e62afa94892b85f027c78bb6600, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733116651060 2024-12-02T05:17:33,087 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bde1e7ca1364c2ab53638304e78c2c5, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733116651060 2024-12-02T05:17:33,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T05:17:33,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:33,098 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T05:17:33,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:33,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:33,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:33,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:33,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:33,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:33,103 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T05:17:33,113 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffa3b22adefa33dc88a53908588e6f34#B#compaction#159 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:33,113 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/e5fcf008006c4504812c70cf78b0cf33 is 50, key is test_row_0/B:col10/1733116652212/Put/seqid=0 2024-12-02T05:17:33,113 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120273831040c9644e78bf7a1f784136f66c_ffa3b22adefa33dc88a53908588e6f34 store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:33,117 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120273831040c9644e78bf7a1f784136f66c_ffa3b22adefa33dc88a53908588e6f34, store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:33,118 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120273831040c9644e78bf7a1f784136f66c_ffa3b22adefa33dc88a53908588e6f34 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:33,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026033fd0b268843d8900bedec1fa986ef_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116652237/Put/seqid=0 2024-12-02T05:17:33,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742016_1192 (size=12915) 2024-12-02T05:17:33,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742017_1193 (size=4469) 2024-12-02T05:17:33,141 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffa3b22adefa33dc88a53908588e6f34#A#compaction#158 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:33,141 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/e5280a81ea814932894b13e9dda8a278 is 175, key is test_row_0/A:col10/1733116652212/Put/seqid=0 2024-12-02T05:17:33,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742018_1194 (size=12454) 2024-12-02T05:17:33,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742019_1195 (size=31869) 2024-12-02T05:17:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:33,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:33,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T05:17:33,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116713417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:33,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116713417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:33,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:33,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116713527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116713527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:33,545 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/e5fcf008006c4504812c70cf78b0cf33 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/e5fcf008006c4504812c70cf78b0cf33 2024-12-02T05:17:33,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:33,551 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026033fd0b268843d8900bedec1fa986ef_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026033fd0b268843d8900bedec1fa986ef_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:33,552 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in ffa3b22adefa33dc88a53908588e6f34/B of ffa3b22adefa33dc88a53908588e6f34 into e5fcf008006c4504812c70cf78b0cf33(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:33,552 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:33,553 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., storeName=ffa3b22adefa33dc88a53908588e6f34/B, priority=9, startTime=1733116653083; duration=0sec 2024-12-02T05:17:33,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/dbb19846a9da4d97a9dba4630148afc7, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:33,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/dbb19846a9da4d97a9dba4630148afc7 is 175, key is test_row_0/A:col10/1733116652237/Put/seqid=0 2024-12-02T05:17:33,554 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:33,554 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffa3b22adefa33dc88a53908588e6f34:B 2024-12-02T05:17:33,554 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-12-02T05:17:33,556 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 85633 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-12-02T05:17:33,557 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): ffa3b22adefa33dc88a53908588e6f34/C is initiating minor compaction (all files) 2024-12-02T05:17:33,557 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffa3b22adefa33dc88a53908588e6f34/C in TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:33,557 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/cfe17072c8d949d6b98a27f70e46e314, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/100cc090bad1421fa34ac472d1187d79, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/963377caf10a400e870d2feddd57d5fb, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/e519579efa86469ea4239e7017f86607, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/9bbc10620d3a480584803b38c90ca9fb, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/8cc1e86f6581445a941b0020d28dbf9c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/fa9dfbf351494378a823ebf899ecf1e0] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp, totalSize=83.6 K 2024-12-02T05:17:33,557 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfe17072c8d949d6b98a27f70e46e314, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733116643045 2024-12-02T05:17:33,559 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 100cc090bad1421fa34ac472d1187d79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733116643701 2024-12-02T05:17:33,559 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 963377caf10a400e870d2feddd57d5fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733116644375 2024-12-02T05:17:33,560 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting e519579efa86469ea4239e7017f86607, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733116645530 2024-12-02T05:17:33,560 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bbc10620d3a480584803b38c90ca9fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733116646714 2024-12-02T05:17:33,561 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cc1e86f6581445a941b0020d28dbf9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733116648886 2024-12-02T05:17:33,561 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa9dfbf351494378a823ebf899ecf1e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733116651060 2024-12-02T05:17:33,562 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/e5280a81ea814932894b13e9dda8a278 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e5280a81ea814932894b13e9dda8a278 2024-12-02T05:17:33,568 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in ffa3b22adefa33dc88a53908588e6f34/A of ffa3b22adefa33dc88a53908588e6f34 into e5280a81ea814932894b13e9dda8a278(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:33,568 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:33,568 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., storeName=ffa3b22adefa33dc88a53908588e6f34/A, priority=9, startTime=1733116653083; duration=0sec 2024-12-02T05:17:33,568 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:33,569 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffa3b22adefa33dc88a53908588e6f34:A 2024-12-02T05:17:33,578 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffa3b22adefa33dc88a53908588e6f34#C#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:33,579 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/c0ef102890de45cfaa76b9abb3a66e5a is 50, key is test_row_0/C:col10/1733116652212/Put/seqid=0 2024-12-02T05:17:33,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742020_1196 (size=31255) 2024-12-02T05:17:33,592 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/dbb19846a9da4d97a9dba4630148afc7 2024-12-02T05:17:33,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742021_1197 (size=12915) 2024-12-02T05:17:33,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/afdf3c394c854d67a2a187190ec2fc5e is 50, key is test_row_0/B:col10/1733116652237/Put/seqid=0 2024-12-02T05:17:33,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742022_1198 (size=12301) 2024-12-02T05:17:33,634 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/afdf3c394c854d67a2a187190ec2fc5e 2024-12-02T05:17:33,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/794df26cb9ba446bad723c072991b8e8 is 50, key is test_row_0/C:col10/1733116652237/Put/seqid=0 2024-12-02T05:17:33,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742023_1199 (size=12301) 2024-12-02T05:17:33,650 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/794df26cb9ba446bad723c072991b8e8 2024-12-02T05:17:33,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/dbb19846a9da4d97a9dba4630148afc7 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/dbb19846a9da4d97a9dba4630148afc7 2024-12-02T05:17:33,661 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/dbb19846a9da4d97a9dba4630148afc7, entries=150, sequenceid=291, filesize=30.5 K 2024-12-02T05:17:33,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/afdf3c394c854d67a2a187190ec2fc5e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/afdf3c394c854d67a2a187190ec2fc5e 2024-12-02T05:17:33,670 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/afdf3c394c854d67a2a187190ec2fc5e, entries=150, sequenceid=291, filesize=12.0 K 2024-12-02T05:17:33,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/794df26cb9ba446bad723c072991b8e8 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/794df26cb9ba446bad723c072991b8e8 2024-12-02T05:17:33,681 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/794df26cb9ba446bad723c072991b8e8, entries=150, sequenceid=291, filesize=12.0 K 2024-12-02T05:17:33,682 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for ffa3b22adefa33dc88a53908588e6f34 in 584ms, sequenceid=291, compaction requested=false 2024-12-02T05:17:33,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:33,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:33,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-02T05:17:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-02T05:17:33,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-02T05:17:33,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 894 msec 2024-12-02T05:17:33,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 899 msec 2024-12-02T05:17:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:33,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T05:17:33,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:33,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:33,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:33,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:33,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:33,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:33,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120292292870cfdb4d9ab74b2201da3f7efd_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116653740/Put/seqid=0 2024-12-02T05:17:33,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:33,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116713766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116713766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:33,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742024_1200 (size=17534) 2024-12-02T05:17:33,800 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:33,805 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120292292870cfdb4d9ab74b2201da3f7efd_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120292292870cfdb4d9ab74b2201da3f7efd_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:33,807 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/d8a78d23223b463a8a800d0f7d7330c2, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:33,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/d8a78d23223b463a8a800d0f7d7330c2 is 175, key is test_row_0/A:col10/1733116653740/Put/seqid=0 2024-12-02T05:17:33,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742025_1201 (size=48639) 2024-12-02T05:17:33,835 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/d8a78d23223b463a8a800d0f7d7330c2 2024-12-02T05:17:33,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/d73a10e2bc264d4a862649e79771a1fa is 50, key is test_row_0/B:col10/1733116653740/Put/seqid=0 2024-12-02T05:17:33,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742026_1202 (size=12301) 2024-12-02T05:17:33,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116713878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116713878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T05:17:33,928 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:33,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-02T05:17:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T05:17:33,931 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:33,932 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:33,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:34,016 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/c0ef102890de45cfaa76b9abb3a66e5a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/c0ef102890de45cfaa76b9abb3a66e5a 2024-12-02T05:17:34,022 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in ffa3b22adefa33dc88a53908588e6f34/C of ffa3b22adefa33dc88a53908588e6f34 into c0ef102890de45cfaa76b9abb3a66e5a(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:34,022 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:34,022 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., storeName=ffa3b22adefa33dc88a53908588e6f34/C, priority=9, startTime=1733116653083; duration=0sec 2024-12-02T05:17:34,022 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:34,022 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffa3b22adefa33dc88a53908588e6f34:C 2024-12-02T05:17:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T05:17:34,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T05:17:34,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:34,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:34,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:34,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:34,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:34,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:34,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116714088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:34,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116714088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:34,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T05:17:34,318 INFO [AsyncFSWAL-0-hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData-prefix:f49461030c22,42731,1733116593060 {}] wal.AbstractFSWAL(1368): Slow sync cost: 230 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34519,DS-5a5c0e22-a56e-4f32-ae1e-a90082acf4bf,DISK]] 2024-12-02T05:17:34,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/d73a10e2bc264d4a862649e79771a1fa 2024-12-02T05:17:34,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/5a89393456a74c03b5274cea26a1623f is 50, key is test_row_0/C:col10/1733116653740/Put/seqid=0 2024-12-02T05:17:34,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742027_1203 (size=12301) 2024-12-02T05:17:34,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:34,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49854 deadline: 1733116714397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:34,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:34,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:34,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:34,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733116714397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:34,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:34,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:34,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:34,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:17:34,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T05:17:34,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:34,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:34,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:34,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:34,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:34,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:34,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T05:17:34,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T05:17:34,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:34,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:34,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:34,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:34,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:34,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/5a89393456a74c03b5274cea26a1623f 2024-12-02T05:17:34,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/d8a78d23223b463a8a800d0f7d7330c2 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/d8a78d23223b463a8a800d0f7d7330c2 2024-12-02T05:17:34,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/d8a78d23223b463a8a800d0f7d7330c2, entries=250, sequenceid=313, filesize=47.5 K 2024-12-02T05:17:34,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/d73a10e2bc264d4a862649e79771a1fa as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/d73a10e2bc264d4a862649e79771a1fa 2024-12-02T05:17:34,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/d73a10e2bc264d4a862649e79771a1fa, entries=150, sequenceid=313, filesize=12.0 K 2024-12-02T05:17:34,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/5a89393456a74c03b5274cea26a1623f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/5a89393456a74c03b5274cea26a1623f 2024-12-02T05:17:34,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/5a89393456a74c03b5274cea26a1623f, entries=150, sequenceid=313, filesize=12.0 K 2024-12-02T05:17:34,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ffa3b22adefa33dc88a53908588e6f34 in 1011ms, sequenceid=313, compaction requested=true 2024-12-02T05:17:34,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:34,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffa3b22adefa33dc88a53908588e6f34:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:17:34,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:34,752 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:34,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffa3b22adefa33dc88a53908588e6f34:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:17:34,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:34,752 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:34,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffa3b22adefa33dc88a53908588e6f34:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:17:34,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:34,753 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111763 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:34,753 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:34,753 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): ffa3b22adefa33dc88a53908588e6f34/A is initiating minor compaction (all files) 2024-12-02T05:17:34,753 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ffa3b22adefa33dc88a53908588e6f34/B is initiating minor compaction (all files) 2024-12-02T05:17:34,753 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffa3b22adefa33dc88a53908588e6f34/B in TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:34,753 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffa3b22adefa33dc88a53908588e6f34/A in TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:34,753 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e5280a81ea814932894b13e9dda8a278, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/dbb19846a9da4d97a9dba4630148afc7, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/d8a78d23223b463a8a800d0f7d7330c2] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp, totalSize=109.1 K 2024-12-02T05:17:34,753 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/e5fcf008006c4504812c70cf78b0cf33, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/afdf3c394c854d67a2a187190ec2fc5e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/d73a10e2bc264d4a862649e79771a1fa] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp, totalSize=36.6 K 2024-12-02T05:17:34,753 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:34,754 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e5280a81ea814932894b13e9dda8a278, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/dbb19846a9da4d97a9dba4630148afc7, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/d8a78d23223b463a8a800d0f7d7330c2] 2024-12-02T05:17:34,754 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5280a81ea814932894b13e9dda8a278, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733116651060 2024-12-02T05:17:34,754 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting e5fcf008006c4504812c70cf78b0cf33, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733116651060 2024-12-02T05:17:34,754 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbb19846a9da4d97a9dba4630148afc7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733116652237 2024-12-02T05:17:34,754 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting afdf3c394c854d67a2a187190ec2fc5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733116652237 2024-12-02T05:17:34,755 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8a78d23223b463a8a800d0f7d7330c2, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733116653399 2024-12-02T05:17:34,755 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting d73a10e2bc264d4a862649e79771a1fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733116653415 2024-12-02T05:17:34,761 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:34,762 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffa3b22adefa33dc88a53908588e6f34#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:34,763 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/54236fdaff7540bea2bfdc7b2a2b67bd is 50, key is test_row_0/B:col10/1733116653740/Put/seqid=0 2024-12-02T05:17:34,766 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120261bac6c5b1e64026b2c1455ee020a36d_ffa3b22adefa33dc88a53908588e6f34 store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:34,768 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120261bac6c5b1e64026b2c1455ee020a36d_ffa3b22adefa33dc88a53908588e6f34, store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:34,768 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120261bac6c5b1e64026b2c1455ee020a36d_ffa3b22adefa33dc88a53908588e6f34 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:34,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742028_1204 (size=13017) 2024-12-02T05:17:34,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T05:17:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:34,775 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T05:17:34,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:34,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:34,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:34,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:34,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:34,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:34,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742029_1205 (size=4469) 2024-12-02T05:17:34,781 INFO [Thread-619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-619. 2024-12-02T05:17:34,781 DEBUG [Thread-619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:34,781 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:34,781 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:34,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:34,782 INFO [Thread-623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-623. 2024-12-02T05:17:34,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202eec33afae771491ba389d2de1fec4fe7_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116653763/Put/seqid=0 2024-12-02T05:17:34,782 DEBUG [Thread-623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:34,782 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:34,783 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:34,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:34,783 INFO [Thread-621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-621. 2024-12-02T05:17:34,783 DEBUG [Thread-621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:34,783 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:34,783 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:34,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:34,787 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-12-02T05:17:34,787 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:34,787 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:34,787 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:34,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:34,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742030_1206 (size=12454) 2024-12-02T05:17:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:34,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. as already flushing 2024-12-02T05:17:34,911 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-12-02T05:17:34,911 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:34,912 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:34,912 INFO [Thread-608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-608. 2024-12-02T05:17:34,912 DEBUG [Thread-608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:34,912 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:34,912 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:34,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:34,912 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:34,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T05:17:35,180 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffa3b22adefa33dc88a53908588e6f34#A#compaction#167 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:35,180 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/e79c3d7ea5f541ebb45b1a754ad239b6 is 175, key is test_row_0/A:col10/1733116653740/Put/seqid=0 2024-12-02T05:17:35,181 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/54236fdaff7540bea2bfdc7b2a2b67bd as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/54236fdaff7540bea2bfdc7b2a2b67bd 2024-12-02T05:17:35,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742031_1207 (size=31971) 2024-12-02T05:17:35,187 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffa3b22adefa33dc88a53908588e6f34/B of ffa3b22adefa33dc88a53908588e6f34 into 54236fdaff7540bea2bfdc7b2a2b67bd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:35,187 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:35,187 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., storeName=ffa3b22adefa33dc88a53908588e6f34/B, priority=13, startTime=1733116654752; duration=0sec 2024-12-02T05:17:35,187 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:35,187 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffa3b22adefa33dc88a53908588e6f34:B 2024-12-02T05:17:35,187 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:35,188 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:35,189 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ffa3b22adefa33dc88a53908588e6f34/C is initiating minor compaction (all files) 2024-12-02T05:17:35,189 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffa3b22adefa33dc88a53908588e6f34/C in TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:35,189 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/c0ef102890de45cfaa76b9abb3a66e5a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/794df26cb9ba446bad723c072991b8e8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/5a89393456a74c03b5274cea26a1623f] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp, totalSize=36.6 K 2024-12-02T05:17:35,189 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting c0ef102890de45cfaa76b9abb3a66e5a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733116651060 2024-12-02T05:17:35,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:35,189 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 794df26cb9ba446bad723c072991b8e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733116652237 2024-12-02T05:17:35,190 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a89393456a74c03b5274cea26a1623f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733116653415 2024-12-02T05:17:35,193 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202eec33afae771491ba389d2de1fec4fe7_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202eec33afae771491ba389d2de1fec4fe7_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:35,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/3a6ffee749e74c3e84586cbb35aabef5, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:35,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/3a6ffee749e74c3e84586cbb35aabef5 is 175, key is test_row_0/A:col10/1733116653763/Put/seqid=0 2024-12-02T05:17:35,196 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffa3b22adefa33dc88a53908588e6f34#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:35,197 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/c06e0408f63244e5a5992a49b4fccdd9 is 50, key is test_row_0/C:col10/1733116653740/Put/seqid=0 2024-12-02T05:17:35,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742032_1208 (size=31255) 2024-12-02T05:17:35,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742033_1209 (size=13017) 2024-12-02T05:17:35,597 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/e79c3d7ea5f541ebb45b1a754ad239b6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e79c3d7ea5f541ebb45b1a754ad239b6 2024-12-02T05:17:35,599 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/3a6ffee749e74c3e84586cbb35aabef5 2024-12-02T05:17:35,602 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffa3b22adefa33dc88a53908588e6f34/A of ffa3b22adefa33dc88a53908588e6f34 into e79c3d7ea5f541ebb45b1a754ad239b6(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:35,602 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:35,602 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., storeName=ffa3b22adefa33dc88a53908588e6f34/A, priority=13, startTime=1733116654752; duration=0sec 2024-12-02T05:17:35,602 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:35,603 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffa3b22adefa33dc88a53908588e6f34:A 2024-12-02T05:17:35,606 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/c06e0408f63244e5a5992a49b4fccdd9 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/c06e0408f63244e5a5992a49b4fccdd9 2024-12-02T05:17:35,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/8a0f68decf3a4d949bcaf035f0305cb1 is 50, key is test_row_0/B:col10/1733116653763/Put/seqid=0 2024-12-02T05:17:35,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742034_1210 (size=12301) 2024-12-02T05:17:35,613 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffa3b22adefa33dc88a53908588e6f34/C of ffa3b22adefa33dc88a53908588e6f34 into c06e0408f63244e5a5992a49b4fccdd9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:35,613 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:35,613 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., storeName=ffa3b22adefa33dc88a53908588e6f34/C, priority=13, startTime=1733116654752; duration=0sec 2024-12-02T05:17:35,613 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:35,613 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffa3b22adefa33dc88a53908588e6f34:C 2024-12-02T05:17:36,013 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/8a0f68decf3a4d949bcaf035f0305cb1 2024-12-02T05:17:36,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/a563162e290e4d4095d7f3579b22eb8a is 50, key is test_row_0/C:col10/1733116653763/Put/seqid=0 2024-12-02T05:17:36,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742035_1211 (size=12301) 2024-12-02T05:17:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T05:17:36,425 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/a563162e290e4d4095d7f3579b22eb8a 2024-12-02T05:17:36,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/3a6ffee749e74c3e84586cbb35aabef5 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/3a6ffee749e74c3e84586cbb35aabef5 2024-12-02T05:17:36,438 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/3a6ffee749e74c3e84586cbb35aabef5, entries=150, sequenceid=326, filesize=30.5 K 2024-12-02T05:17:36,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/8a0f68decf3a4d949bcaf035f0305cb1 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/8a0f68decf3a4d949bcaf035f0305cb1 2024-12-02T05:17:36,446 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/8a0f68decf3a4d949bcaf035f0305cb1, entries=150, sequenceid=326, filesize=12.0 K 2024-12-02T05:17:36,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/a563162e290e4d4095d7f3579b22eb8a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/a563162e290e4d4095d7f3579b22eb8a 2024-12-02T05:17:36,452 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/a563162e290e4d4095d7f3579b22eb8a, entries=150, sequenceid=326, filesize=12.0 K 2024-12-02T05:17:36,453 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=13.42 KB/13740 for ffa3b22adefa33dc88a53908588e6f34 in 1678ms, sequenceid=326, compaction requested=false 2024-12-02T05:17:36,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for ffa3b22adefa33dc88a53908588e6f34: 2024-12-02T05:17:36,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:36,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-02T05:17:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-02T05:17:36,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-02T05:17:36,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5230 sec 2024-12-02T05:17:36,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.5290 sec 2024-12-02T05:17:38,080 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-12-02T05:17:38,081 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:38,081 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:38,081 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:38,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T05:17:38,088 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:39,189 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-12-02T05:17:39,189 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:39,189 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:39,189 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:39,189 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:41,970 INFO [Thread-616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-616. 2024-12-02T05:17:41,970 DEBUG [Thread-616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:17:41,970 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:41,970 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:41,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T05:17:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-12-02T05:17:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-12-02T05:17:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-12-02T05:17:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-12-02T05:17:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-12-02T05:17:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T05:17:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4014 2024-12-02T05:17:41,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4020 2024-12-02T05:17:41,971 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T05:17:41,971 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1747 2024-12-02T05:17:41,971 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5241 rows 2024-12-02T05:17:41,971 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1754 2024-12-02T05:17:41,971 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5260 rows 2024-12-02T05:17:41,971 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T05:17:41,971 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T05:17:41,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:41,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:41,971 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:17:41,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T05:17:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T05:17:41,975 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116661975"}]},"ts":"1733116661975"} 2024-12-02T05:17:41,976 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T05:17:41,976 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T05:17:41,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T05:17:41,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffa3b22adefa33dc88a53908588e6f34, UNASSIGN}] 2024-12-02T05:17:41,979 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffa3b22adefa33dc88a53908588e6f34, UNASSIGN 2024-12-02T05:17:41,980 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=ffa3b22adefa33dc88a53908588e6f34, regionState=CLOSING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:17:41,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffa3b22adefa33dc88a53908588e6f34, UNASSIGN because future has completed 2024-12-02T05:17:41,981 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T05:17:41,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:17:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T05:17:42,133 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:42,134 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T05:17:42,134 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing ffa3b22adefa33dc88a53908588e6f34, disabling compactions & flushes 2024-12-02T05:17:42,134 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:42,134 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:42,134 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. after waiting 0 ms 2024-12-02T05:17:42,134 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:42,134 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing ffa3b22adefa33dc88a53908588e6f34 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T05:17:42,134 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=A 2024-12-02T05:17:42,134 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:42,134 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=B 2024-12-02T05:17:42,134 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:42,134 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffa3b22adefa33dc88a53908588e6f34, store=C 2024-12-02T05:17:42,134 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:42,178 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ff4c0052695949cca664253687d34893_ffa3b22adefa33dc88a53908588e6f34 is 50, key is test_row_0/A:col10/1733116661968/Put/seqid=0 2024-12-02T05:17:42,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742036_1212 (size=12454) 2024-12-02T05:17:42,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T05:17:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T05:17:42,599 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:42,604 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ff4c0052695949cca664253687d34893_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ff4c0052695949cca664253687d34893_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:42,606 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/4f57ae6c91f9477fb3e63758f665aa9c, store: [table=TestAcidGuarantees family=A region=ffa3b22adefa33dc88a53908588e6f34] 2024-12-02T05:17:42,606 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/4f57ae6c91f9477fb3e63758f665aa9c is 175, key is test_row_0/A:col10/1733116661968/Put/seqid=0 2024-12-02T05:17:42,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742037_1213 (size=31255) 2024-12-02T05:17:43,012 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/4f57ae6c91f9477fb3e63758f665aa9c 2024-12-02T05:17:43,025 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/eb657166ec434a9d95ba2d2cba235a60 is 50, key is test_row_0/B:col10/1733116661968/Put/seqid=0 2024-12-02T05:17:43,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742038_1214 (size=12301) 2024-12-02T05:17:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T05:17:43,429 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/eb657166ec434a9d95ba2d2cba235a60 2024-12-02T05:17:43,437 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/f900a1008b7b4a5686f24c09dc8ae088 is 50, key is test_row_0/C:col10/1733116661968/Put/seqid=0 2024-12-02T05:17:43,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742039_1215 (size=12301) 2024-12-02T05:17:43,842 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/f900a1008b7b4a5686f24c09dc8ae088 2024-12-02T05:17:43,848 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/A/4f57ae6c91f9477fb3e63758f665aa9c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/4f57ae6c91f9477fb3e63758f665aa9c 2024-12-02T05:17:43,853 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/4f57ae6c91f9477fb3e63758f665aa9c, entries=150, sequenceid=337, filesize=30.5 K 2024-12-02T05:17:43,854 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/B/eb657166ec434a9d95ba2d2cba235a60 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/eb657166ec434a9d95ba2d2cba235a60 2024-12-02T05:17:43,858 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/eb657166ec434a9d95ba2d2cba235a60, entries=150, sequenceid=337, filesize=12.0 K 2024-12-02T05:17:43,859 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/.tmp/C/f900a1008b7b4a5686f24c09dc8ae088 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/f900a1008b7b4a5686f24c09dc8ae088 2024-12-02T05:17:43,864 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/f900a1008b7b4a5686f24c09dc8ae088, entries=150, sequenceid=337, filesize=12.0 K 2024-12-02T05:17:43,865 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ffa3b22adefa33dc88a53908588e6f34 in 1731ms, sequenceid=337, compaction requested=true 2024-12-02T05:17:43,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e2e71b8487c84eb9b7c69963c24623b1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/829c72fcab5d45b5b0057b41b736a850, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/11644da6078747bebee67a856e9cf342, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e6b2187eb5aa494580090b819ae5dfb6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/597a7acdaea049fb89474c4859324e32, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/1dddeb376ec343afb595c93cc9669663, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/78061e250519413f8a3076acdcb391bf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/59a79a9496634e01b66aa55d5fcbd6f4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/85253287918f436c97a7c2831db250bd, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/c79ee38ce3b9400ab218bff7de31dffc, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/79b90d0344854d149f58cd9054b80582, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/dbba756c3a1d476f9405b4a7a52e695f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/c7172f09929a445494e838906cd6a031, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/7174f37b5c494e728075c11acd0590f7, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/af985620a1b74113a80aecb96d463319, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/f3213610b43d4fde98ddf0e2b64f6177, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/3a334263204840b7b1b8c506400b9582, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/8bde1e7ca1364c2ab53638304e78c2c5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e5280a81ea814932894b13e9dda8a278, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/dbb19846a9da4d97a9dba4630148afc7, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/d8a78d23223b463a8a800d0f7d7330c2] to archive 2024-12-02T05:17:43,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:17:43,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e2e71b8487c84eb9b7c69963c24623b1 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e2e71b8487c84eb9b7c69963c24623b1 2024-12-02T05:17:43,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/829c72fcab5d45b5b0057b41b736a850 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/829c72fcab5d45b5b0057b41b736a850 2024-12-02T05:17:43,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/11644da6078747bebee67a856e9cf342 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/11644da6078747bebee67a856e9cf342 2024-12-02T05:17:43,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e6b2187eb5aa494580090b819ae5dfb6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e6b2187eb5aa494580090b819ae5dfb6 2024-12-02T05:17:43,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/597a7acdaea049fb89474c4859324e32 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/597a7acdaea049fb89474c4859324e32 2024-12-02T05:17:43,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/1dddeb376ec343afb595c93cc9669663 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/1dddeb376ec343afb595c93cc9669663 2024-12-02T05:17:43,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/78061e250519413f8a3076acdcb391bf to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/78061e250519413f8a3076acdcb391bf 2024-12-02T05:17:43,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/59a79a9496634e01b66aa55d5fcbd6f4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/59a79a9496634e01b66aa55d5fcbd6f4 2024-12-02T05:17:43,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/85253287918f436c97a7c2831db250bd to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/85253287918f436c97a7c2831db250bd 2024-12-02T05:17:43,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/c79ee38ce3b9400ab218bff7de31dffc to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/c79ee38ce3b9400ab218bff7de31dffc 2024-12-02T05:17:43,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/79b90d0344854d149f58cd9054b80582 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/79b90d0344854d149f58cd9054b80582 2024-12-02T05:17:43,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/dbba756c3a1d476f9405b4a7a52e695f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/dbba756c3a1d476f9405b4a7a52e695f 2024-12-02T05:17:43,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/c7172f09929a445494e838906cd6a031 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/c7172f09929a445494e838906cd6a031 2024-12-02T05:17:43,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/7174f37b5c494e728075c11acd0590f7 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/7174f37b5c494e728075c11acd0590f7 2024-12-02T05:17:43,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/af985620a1b74113a80aecb96d463319 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/af985620a1b74113a80aecb96d463319 2024-12-02T05:17:43,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/f3213610b43d4fde98ddf0e2b64f6177 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/f3213610b43d4fde98ddf0e2b64f6177 2024-12-02T05:17:43,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/3a334263204840b7b1b8c506400b9582 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/3a334263204840b7b1b8c506400b9582 2024-12-02T05:17:43,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/8bde1e7ca1364c2ab53638304e78c2c5 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/8bde1e7ca1364c2ab53638304e78c2c5 2024-12-02T05:17:43,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e5280a81ea814932894b13e9dda8a278 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e5280a81ea814932894b13e9dda8a278 2024-12-02T05:17:43,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/dbb19846a9da4d97a9dba4630148afc7 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/dbb19846a9da4d97a9dba4630148afc7 2024-12-02T05:17:43,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/d8a78d23223b463a8a800d0f7d7330c2 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/d8a78d23223b463a8a800d0f7d7330c2 2024-12-02T05:17:43,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/c24fcdf64f95474eb926b87bb86efd70, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/222dc9dcc5d145cd866cc6fb832d9f68, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/2a83930e713f41759f0e247dbc24e319, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/510dff0b1ba246b891b6512f72cb83ac, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/71300e7ca40e4242add027a827ddb9a6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/5507035f74594910b8d6a07dde7692fa, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/f88c4bd1e9bc4c42ae18ea13873c4d17, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/a1a0d61b930240169b0081732b1056c3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/01cd5f2674064729a97ea072b136891e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/771c2486e3c748af8db36839163c2337, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/582b6a6c69d5408ead1c1f1c12928129, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/af0f10f2e5db464a8956493fa6fe534a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/d8a7c0be78bb4089ad021a4394abe1de, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/2ac898353f7a403db65204bfb9fbedbc, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/299301c6762f4221befde06feefe0641, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/3cc432cbecaa4f14b91aa095f32dfed4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/fc5683f7930a4cf1a0470f151b32bf1b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/e5fcf008006c4504812c70cf78b0cf33, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/4ee70e62afa94892b85f027c78bb6600, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/afdf3c394c854d67a2a187190ec2fc5e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/d73a10e2bc264d4a862649e79771a1fa] to archive 2024-12-02T05:17:43,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:17:43,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/c24fcdf64f95474eb926b87bb86efd70 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/c24fcdf64f95474eb926b87bb86efd70 2024-12-02T05:17:43,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/222dc9dcc5d145cd866cc6fb832d9f68 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/222dc9dcc5d145cd866cc6fb832d9f68 2024-12-02T05:17:43,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/2a83930e713f41759f0e247dbc24e319 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/2a83930e713f41759f0e247dbc24e319 2024-12-02T05:17:43,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/510dff0b1ba246b891b6512f72cb83ac to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/510dff0b1ba246b891b6512f72cb83ac 2024-12-02T05:17:43,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/71300e7ca40e4242add027a827ddb9a6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/71300e7ca40e4242add027a827ddb9a6 2024-12-02T05:17:43,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/5507035f74594910b8d6a07dde7692fa to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/5507035f74594910b8d6a07dde7692fa 2024-12-02T05:17:43,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/f88c4bd1e9bc4c42ae18ea13873c4d17 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/f88c4bd1e9bc4c42ae18ea13873c4d17 2024-12-02T05:17:43,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/a1a0d61b930240169b0081732b1056c3 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/a1a0d61b930240169b0081732b1056c3 2024-12-02T05:17:43,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/01cd5f2674064729a97ea072b136891e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/01cd5f2674064729a97ea072b136891e 2024-12-02T05:17:43,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/771c2486e3c748af8db36839163c2337 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/771c2486e3c748af8db36839163c2337 2024-12-02T05:17:43,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/582b6a6c69d5408ead1c1f1c12928129 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/582b6a6c69d5408ead1c1f1c12928129 2024-12-02T05:17:43,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/af0f10f2e5db464a8956493fa6fe534a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/af0f10f2e5db464a8956493fa6fe534a 2024-12-02T05:17:43,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/d8a7c0be78bb4089ad021a4394abe1de to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/d8a7c0be78bb4089ad021a4394abe1de 2024-12-02T05:17:43,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/2ac898353f7a403db65204bfb9fbedbc to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/2ac898353f7a403db65204bfb9fbedbc 2024-12-02T05:17:43,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/299301c6762f4221befde06feefe0641 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/299301c6762f4221befde06feefe0641 2024-12-02T05:17:43,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/3cc432cbecaa4f14b91aa095f32dfed4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/3cc432cbecaa4f14b91aa095f32dfed4 2024-12-02T05:17:43,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/fc5683f7930a4cf1a0470f151b32bf1b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/fc5683f7930a4cf1a0470f151b32bf1b 2024-12-02T05:17:43,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/e5fcf008006c4504812c70cf78b0cf33 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/e5fcf008006c4504812c70cf78b0cf33 2024-12-02T05:17:43,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/4ee70e62afa94892b85f027c78bb6600 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/4ee70e62afa94892b85f027c78bb6600 2024-12-02T05:17:43,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/afdf3c394c854d67a2a187190ec2fc5e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/afdf3c394c854d67a2a187190ec2fc5e 2024-12-02T05:17:43,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/d73a10e2bc264d4a862649e79771a1fa to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/d73a10e2bc264d4a862649e79771a1fa 2024-12-02T05:17:43,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/57e766a45b3f4946a96b78617798adbf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/e48fd93db5ea4d30ba4022bd9e8282a6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/f837a19ef7e049b9996ffe4d637da9ca, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/b9799e649e034706bf6e64d161363b6a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/111a517d7ee44e0683eafc8ef4210afe, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/d012ef187ca14b2ab67029227165f11f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/98941203cae047f1a85718e82a3c7bae, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/3b8824b1016c4f48a23fb5219fc08a3b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/6c12540c8c0843ffb5162a70d7e175f2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/dde2008cf7f6450cbefb9bb52057e524, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/cfe17072c8d949d6b98a27f70e46e314, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/c94d3e1e92824421bc3c320344d681e4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/100cc090bad1421fa34ac472d1187d79, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/963377caf10a400e870d2feddd57d5fb, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/e519579efa86469ea4239e7017f86607, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/9bbc10620d3a480584803b38c90ca9fb, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/8cc1e86f6581445a941b0020d28dbf9c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/c0ef102890de45cfaa76b9abb3a66e5a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/fa9dfbf351494378a823ebf899ecf1e0, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/794df26cb9ba446bad723c072991b8e8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/5a89393456a74c03b5274cea26a1623f] to archive 2024-12-02T05:17:43,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:17:43,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/57e766a45b3f4946a96b78617798adbf to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/57e766a45b3f4946a96b78617798adbf 2024-12-02T05:17:43,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/e48fd93db5ea4d30ba4022bd9e8282a6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/e48fd93db5ea4d30ba4022bd9e8282a6 2024-12-02T05:17:43,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/f837a19ef7e049b9996ffe4d637da9ca to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/f837a19ef7e049b9996ffe4d637da9ca 2024-12-02T05:17:43,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/b9799e649e034706bf6e64d161363b6a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/b9799e649e034706bf6e64d161363b6a 2024-12-02T05:17:43,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/111a517d7ee44e0683eafc8ef4210afe to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/111a517d7ee44e0683eafc8ef4210afe 2024-12-02T05:17:43,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/d012ef187ca14b2ab67029227165f11f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/d012ef187ca14b2ab67029227165f11f 2024-12-02T05:17:43,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/98941203cae047f1a85718e82a3c7bae to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/98941203cae047f1a85718e82a3c7bae 2024-12-02T05:17:43,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/3b8824b1016c4f48a23fb5219fc08a3b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/3b8824b1016c4f48a23fb5219fc08a3b 2024-12-02T05:17:43,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/6c12540c8c0843ffb5162a70d7e175f2 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/6c12540c8c0843ffb5162a70d7e175f2 2024-12-02T05:17:43,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/dde2008cf7f6450cbefb9bb52057e524 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/dde2008cf7f6450cbefb9bb52057e524 2024-12-02T05:17:43,942 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/cfe17072c8d949d6b98a27f70e46e314 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/cfe17072c8d949d6b98a27f70e46e314 2024-12-02T05:17:43,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/c94d3e1e92824421bc3c320344d681e4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/c94d3e1e92824421bc3c320344d681e4 2024-12-02T05:17:43,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/100cc090bad1421fa34ac472d1187d79 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/100cc090bad1421fa34ac472d1187d79 2024-12-02T05:17:43,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/963377caf10a400e870d2feddd57d5fb to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/963377caf10a400e870d2feddd57d5fb 2024-12-02T05:17:43,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/e519579efa86469ea4239e7017f86607 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/e519579efa86469ea4239e7017f86607 2024-12-02T05:17:43,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/9bbc10620d3a480584803b38c90ca9fb to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/9bbc10620d3a480584803b38c90ca9fb 2024-12-02T05:17:43,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/8cc1e86f6581445a941b0020d28dbf9c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/8cc1e86f6581445a941b0020d28dbf9c 2024-12-02T05:17:43,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/c0ef102890de45cfaa76b9abb3a66e5a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/c0ef102890de45cfaa76b9abb3a66e5a 2024-12-02T05:17:43,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/fa9dfbf351494378a823ebf899ecf1e0 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/fa9dfbf351494378a823ebf899ecf1e0 2024-12-02T05:17:43,951 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/794df26cb9ba446bad723c072991b8e8 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/794df26cb9ba446bad723c072991b8e8 2024-12-02T05:17:43,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/5a89393456a74c03b5274cea26a1623f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/5a89393456a74c03b5274cea26a1623f 2024-12-02T05:17:43,958 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/recovered.edits/340.seqid, newMaxSeqId=340, maxSeqId=4 2024-12-02T05:17:43,959 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. 2024-12-02T05:17:43,959 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for ffa3b22adefa33dc88a53908588e6f34: Waiting for close lock at 1733116662134Running coprocessor pre-close hooks at 1733116662134Disabling compacts and flushes for region at 1733116662134Disabling writes for close at 1733116662134Obtaining lock to block concurrent updates at 1733116662134Preparing flush snapshotting stores in ffa3b22adefa33dc88a53908588e6f34 at 1733116662134Finished memstore snapshotting TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733116662134Flushing stores of TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34. at 1733116662135 (+1 ms)Flushing ffa3b22adefa33dc88a53908588e6f34/A: creating writer at 1733116662135Flushing mob file ffa3b22adefa33dc88a53908588e6f34/A: appending metadata at 1733116662178 (+43 ms)Flushing mob file ffa3b22adefa33dc88a53908588e6f34/A: closing flushed file at 1733116662178Flushing ffa3b22adefa33dc88a53908588e6f34/A: appending metadata at 1733116662606 (+428 ms)Flushing ffa3b22adefa33dc88a53908588e6f34/A: closing flushed file at 1733116662606Flushing ffa3b22adefa33dc88a53908588e6f34/B: creating writer at 1733116663020 (+414 ms)Flushing ffa3b22adefa33dc88a53908588e6f34/B: appending metadata at 1733116663025 (+5 ms)Flushing ffa3b22adefa33dc88a53908588e6f34/B: closing flushed file at 1733116663025Flushing ffa3b22adefa33dc88a53908588e6f34/C: creating writer at 1733116663434 (+409 ms)Flushing ffa3b22adefa33dc88a53908588e6f34/C: appending metadata at 1733116663437 (+3 ms)Flushing ffa3b22adefa33dc88a53908588e6f34/C: closing flushed file at 1733116663437Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ce70a93: reopening flushed file at 1733116663847 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ef104c4: reopening flushed file at 1733116663853 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72fc7676: reopening flushed file at 1733116663859 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ffa3b22adefa33dc88a53908588e6f34 in 1731ms, sequenceid=337, compaction requested=true at 1733116663865 (+6 ms)Writing region close event to WAL at 1733116663954 (+89 ms)Running coprocessor post-close hooks at 1733116663959 (+5 ms)Closed at 1733116663959 2024-12-02T05:17:43,961 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:43,962 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=ffa3b22adefa33dc88a53908588e6f34, regionState=CLOSED 2024-12-02T05:17:43,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:17:43,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-02T05:17:43,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure ffa3b22adefa33dc88a53908588e6f34, server=f49461030c22,46495,1733116594459 in 1.9830 sec 2024-12-02T05:17:43,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-02T05:17:43,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffa3b22adefa33dc88a53908588e6f34, UNASSIGN in 1.9880 sec 2024-12-02T05:17:43,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-02T05:17:43,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9920 sec 2024-12-02T05:17:43,972 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116663972"}]},"ts":"1733116663972"} 2024-12-02T05:17:43,974 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T05:17:43,974 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T05:17:43,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0030 sec 2024-12-02T05:17:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T05:17:44,118 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:44,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T05:17:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:44,124 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-02T05:17:44,126 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:44,131 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,135 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/recovered.edits] 2024-12-02T05:17:44,138 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/3a6ffee749e74c3e84586cbb35aabef5 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/3a6ffee749e74c3e84586cbb35aabef5 2024-12-02T05:17:44,140 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/4f57ae6c91f9477fb3e63758f665aa9c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/4f57ae6c91f9477fb3e63758f665aa9c 2024-12-02T05:17:44,143 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e79c3d7ea5f541ebb45b1a754ad239b6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/A/e79c3d7ea5f541ebb45b1a754ad239b6 2024-12-02T05:17:44,146 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/54236fdaff7540bea2bfdc7b2a2b67bd to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/54236fdaff7540bea2bfdc7b2a2b67bd 2024-12-02T05:17:44,147 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/8a0f68decf3a4d949bcaf035f0305cb1 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/8a0f68decf3a4d949bcaf035f0305cb1 2024-12-02T05:17:44,148 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/eb657166ec434a9d95ba2d2cba235a60 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/B/eb657166ec434a9d95ba2d2cba235a60 2024-12-02T05:17:44,150 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/a563162e290e4d4095d7f3579b22eb8a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/a563162e290e4d4095d7f3579b22eb8a 2024-12-02T05:17:44,151 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/c06e0408f63244e5a5992a49b4fccdd9 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/c06e0408f63244e5a5992a49b4fccdd9 2024-12-02T05:17:44,152 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/f900a1008b7b4a5686f24c09dc8ae088 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/C/f900a1008b7b4a5686f24c09dc8ae088 2024-12-02T05:17:44,154 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/recovered.edits/340.seqid to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34/recovered.edits/340.seqid 2024-12-02T05:17:44,154 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,154 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T05:17:44,155 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T05:17:44,155 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T05:17:44,159 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120207a54ee1411749019b11ad23c896598c_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120207a54ee1411749019b11ad23c896598c_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,160 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202207bcc37202b4ec9a2209eceb98ea49f_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202207bcc37202b4ec9a2209eceb98ea49f_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,161 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022aa6256252da412a80706b92e5510700_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022aa6256252da412a80706b92e5510700_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,162 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024366ad00188e4e88a2e5f58711993912_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024366ad00188e4e88a2e5f58711993912_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,163 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024feec190ce9b48178a604c5baf3c4300_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024feec190ce9b48178a604c5baf3c4300_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,164 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202517f39c113a844a9881c8c1466847e9d_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202517f39c113a844a9881c8c1466847e9d_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,165 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026033fd0b268843d8900bedec1fa986ef_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026033fd0b268843d8900bedec1fa986ef_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,166 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120264c77214d48f4df19ab1f3f2eafb3cad_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120264c77214d48f4df19ab1f3f2eafb3cad_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,167 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202692d98bc2d9449b38879979e7bb4e4d8_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202692d98bc2d9449b38879979e7bb4e4d8_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,169 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026ac4fc75d55c4aacb6bda5300eb46861_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026ac4fc75d55c4aacb6bda5300eb46861_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,170 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120292292870cfdb4d9ab74b2201da3f7efd_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120292292870cfdb4d9ab74b2201da3f7efd_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,171 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b160eabde3474092923e841021cb4319_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b160eabde3474092923e841021cb4319_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,172 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c91ee6e3b08046c5bd8641eb4eda2566_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c91ee6e3b08046c5bd8641eb4eda2566_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,173 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d4e445bb8bd74805ba2fd44811a84732_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d4e445bb8bd74805ba2fd44811a84732_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,174 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d655846101ff4040856b907779e2f3e1_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d655846101ff4040856b907779e2f3e1_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,175 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202da3892d6862f4aaba761b2196a59c048_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202da3892d6862f4aaba761b2196a59c048_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,176 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dec27cf89be741498dae61493a9b5bdc_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dec27cf89be741498dae61493a9b5bdc_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,176 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202eec33afae771491ba389d2de1fec4fe7_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202eec33afae771491ba389d2de1fec4fe7_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,177 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ff4c0052695949cca664253687d34893_ffa3b22adefa33dc88a53908588e6f34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ff4c0052695949cca664253687d34893_ffa3b22adefa33dc88a53908588e6f34 2024-12-02T05:17:44,178 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T05:17:44,180 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:44,182 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T05:17:44,184 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T05:17:44,185 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:44,185 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T05:17:44,185 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733116664185"}]},"ts":"9223372036854775807"} 2024-12-02T05:17:44,187 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T05:17:44,187 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ffa3b22adefa33dc88a53908588e6f34, NAME => 'TestAcidGuarantees,,1733116631362.ffa3b22adefa33dc88a53908588e6f34.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T05:17:44,187 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T05:17:44,187 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733116664187"}]},"ts":"9223372036854775807"} 2024-12-02T05:17:44,189 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T05:17:44,190 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:44,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-12-02T05:17:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-02T05:17:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T05:17:44,228 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:44,235 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249 (was 247) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_359649851_22 at /127.0.0.1:59656 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/cluster_01f97ee8-23a4-f5a0-8cda-7a5425401614/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/cluster_01f97ee8-23a4-f5a0-8cda-7a5425401614/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=568 (was 588), ProcessCount=11 (was 11), AvailableMemoryMB=6928 (was 5987) - AvailableMemoryMB LEAK? - 2024-12-02T05:17:44,243 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=568, ProcessCount=11, AvailableMemoryMB=6927 2024-12-02T05:17:44,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T05:17:44,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T05:17:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T05:17:44,247 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T05:17:44,247 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:44,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-12-02T05:17:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T05:17:44,248 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T05:17:44,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742040_1216 (size=1061) 2024-12-02T05:17:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T05:17:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T05:17:44,660 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c 2024-12-02T05:17:44,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742041_1217 (size=53) 2024-12-02T05:17:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T05:17:45,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:17:45,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2bec4a5721d5a74dad6f846b3202c90f, disabling compactions & flushes 2024-12-02T05:17:45,072 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:45,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:45,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. after waiting 0 ms 2024-12-02T05:17:45,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:45,072 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:45,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2bec4a5721d5a74dad6f846b3202c90f: Waiting for close lock at 1733116665072Disabling compacts and flushes for region at 1733116665072Disabling writes for close at 1733116665072Writing region close event to WAL at 1733116665072Closed at 1733116665072 2024-12-02T05:17:45,074 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T05:17:45,075 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733116665074"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733116665074"}]},"ts":"1733116665074"} 2024-12-02T05:17:45,078 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T05:17:45,079 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T05:17:45,080 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116665079"}]},"ts":"1733116665079"} 2024-12-02T05:17:45,083 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T05:17:45,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bec4a5721d5a74dad6f846b3202c90f, ASSIGN}] 2024-12-02T05:17:45,085 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bec4a5721d5a74dad6f846b3202c90f, ASSIGN 2024-12-02T05:17:45,086 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bec4a5721d5a74dad6f846b3202c90f, ASSIGN; state=OFFLINE, location=f49461030c22,46495,1733116594459; forceNewPlan=false, retain=false 2024-12-02T05:17:45,237 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=2bec4a5721d5a74dad6f846b3202c90f, regionState=OPENING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:17:45,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bec4a5721d5a74dad6f846b3202c90f, ASSIGN because future has completed 2024-12-02T05:17:45,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:17:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T05:17:45,400 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:45,400 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} 2024-12-02T05:17:45,400 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:45,400 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:17:45,400 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:45,400 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:45,402 INFO [StoreOpener-2bec4a5721d5a74dad6f846b3202c90f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:45,404 INFO [StoreOpener-2bec4a5721d5a74dad6f846b3202c90f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:17:45,404 INFO [StoreOpener-2bec4a5721d5a74dad6f846b3202c90f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bec4a5721d5a74dad6f846b3202c90f columnFamilyName A 2024-12-02T05:17:45,405 DEBUG [StoreOpener-2bec4a5721d5a74dad6f846b3202c90f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:45,405 INFO [StoreOpener-2bec4a5721d5a74dad6f846b3202c90f-1 {}] regionserver.HStore(327): Store=2bec4a5721d5a74dad6f846b3202c90f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:17:45,406 INFO [StoreOpener-2bec4a5721d5a74dad6f846b3202c90f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:45,407 INFO [StoreOpener-2bec4a5721d5a74dad6f846b3202c90f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:17:45,408 INFO [StoreOpener-2bec4a5721d5a74dad6f846b3202c90f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bec4a5721d5a74dad6f846b3202c90f columnFamilyName B 2024-12-02T05:17:45,408 DEBUG [StoreOpener-2bec4a5721d5a74dad6f846b3202c90f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:45,409 INFO [StoreOpener-2bec4a5721d5a74dad6f846b3202c90f-1 {}] regionserver.HStore(327): Store=2bec4a5721d5a74dad6f846b3202c90f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:17:45,409 INFO [StoreOpener-2bec4a5721d5a74dad6f846b3202c90f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:45,411 INFO [StoreOpener-2bec4a5721d5a74dad6f846b3202c90f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:17:45,411 INFO [StoreOpener-2bec4a5721d5a74dad6f846b3202c90f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bec4a5721d5a74dad6f846b3202c90f columnFamilyName C 2024-12-02T05:17:45,411 DEBUG [StoreOpener-2bec4a5721d5a74dad6f846b3202c90f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:17:45,412 INFO [StoreOpener-2bec4a5721d5a74dad6f846b3202c90f-1 {}] regionserver.HStore(327): Store=2bec4a5721d5a74dad6f846b3202c90f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:17:45,412 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:45,412 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:45,413 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:45,414 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:45,415 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:45,415 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:45,416 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T05:17:45,417 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:45,419 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T05:17:45,420 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 2bec4a5721d5a74dad6f846b3202c90f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67271950, jitterRate=0.002430170774459839}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T05:17:45,420 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:45,420 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 2bec4a5721d5a74dad6f846b3202c90f: Running coprocessor pre-open hook at 1733116665400Writing region info on filesystem at 1733116665400Initializing all the Stores at 1733116665402 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116665402Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116665402Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116665402Cleaning up temporary data from old regions at 1733116665415 (+13 ms)Running coprocessor post-open hooks at 1733116665420 (+5 ms)Region opened successfully at 1733116665420 2024-12-02T05:17:45,421 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., pid=63, masterSystemTime=1733116665396 2024-12-02T05:17:45,422 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:45,423 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:45,423 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=2bec4a5721d5a74dad6f846b3202c90f, regionState=OPEN, openSeqNum=2, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:17:45,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:17:45,426 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-02T05:17:45,426 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 in 183 msec 2024-12-02T05:17:45,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-02T05:17:45,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bec4a5721d5a74dad6f846b3202c90f, ASSIGN in 342 msec 2024-12-02T05:17:45,428 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T05:17:45,429 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116665428"}]},"ts":"1733116665428"} 2024-12-02T05:17:45,430 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T05:17:45,431 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T05:17:45,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-12-02T05:17:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T05:17:46,407 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:46,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:46,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:46,409 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c70811e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:46,411 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:46,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4364f5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,411 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:46,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:46,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:46,413 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:46,414 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T05:17:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is f49461030c22,42731,1733116593060 2024-12-02T05:17:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f28751e 2024-12-02T05:17:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T05:17:46,416 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T05:17:46,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,418 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:46,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:46,419 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d2d1224, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:46,421 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c0754a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:46,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:46,423 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:46,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:46,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:46,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20605630, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:46,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:46,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:46,424 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a006de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:46,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:46,425 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ca1d8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:46,427 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5239dbbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:46,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:46,428 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e489bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:46,430 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@622fe13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:46,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:46,432 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32cc21fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:46,433 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e8fcdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,434 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:46,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:46,435 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145e21bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:46,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:46,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:46,436 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:46,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57b68bfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:46,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:46,438 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:46,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:46,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:46,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@617faf50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:46,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:46,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:46,440 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@753f21a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,441 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:46,441 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:46,441 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:46,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:46,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:46,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62bea349, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:46,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:46,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:46,443 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b0495b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,444 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:46,444 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:46,445 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f1d6c58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:46,446 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:46,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2903d7e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:17:46,447 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:17:46,448 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:17:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:17:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:17:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ccf3b78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:17:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:17:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:17:46,449 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:17:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bee8362, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:17:46,450 DEBUG [Thread-1029 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:46,450 DEBUG [Thread-1046 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:46,450 DEBUG [Thread-1035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:46,451 DEBUG [Thread-1031 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:46,451 DEBUG [Thread-1044 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:46,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:46,452 DEBUG [Thread-1027 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:46,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:46,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:46,452 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:46,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:46,452 DEBUG [Thread-1033 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:46,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:46,452 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:46,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:46,452 DEBUG [Thread-1042 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:46,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:46,453 DEBUG [Thread-1038 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:46,453 DEBUG [Thread-1040 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:17:46,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:46,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:46,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:46,454 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:46,454 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:46,454 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-02T05:17:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T05:17:46,455 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:46,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:46,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:46,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:17:46,457 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:46,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:46,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:17:46,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:17:46,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:46,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:46,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:46,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:17:46,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:17:46,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:46,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:46,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:17:46,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:17:46,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:17:46,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:17:46,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:17:46,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:17:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:17:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:17:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:46,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T05:17:46,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:46,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:46,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:46,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:46,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:46,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:46,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116666482, max timestamp=1733116666488Num uniques 859; 2024-12-02T05:17:46,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:17:46,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116666482, max timestamp=1733116666488Num uniques 859; 2024-12-02T05:17:46,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116666482, max timestamp=1733116666488Num uniques 859; 2024-12-02T05:17:46,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:17:46,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:17:46,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/6a65ffe569684ec48a15453f39d9bdfc is 50, key is test_row_0/A:col10/1733116666465/Put/seqid=0 2024-12-02T05:17:46,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742042_1218 (size=12001) 2024-12-02T05:17:46,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:46,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116726511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116726511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:46,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:46,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116726513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:46,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:46,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:46,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:46,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49568 deadline: 1733116726517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:46,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116726518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T05:17:46,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T05:17:46,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:46,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:46,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:46,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:46,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:46,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116726627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:46,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:46,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116726627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116726627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:46,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116726627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:46,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49568 deadline: 1733116726627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:46,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T05:17:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:46,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T05:17:46,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116726837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:46,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:46,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116726838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:46,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:46,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:46,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116726838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116726838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:46,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49568 deadline: 1733116726848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:46,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/6a65ffe569684ec48a15453f39d9bdfc 2024-12-02T05:17:46,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T05:17:46,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:46,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:46,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:46,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:46,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:46,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/26d3062617f74cc89f24d0dbcd4f163d is 50, key is test_row_0/B:col10/1733116666465/Put/seqid=0 2024-12-02T05:17:46,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742043_1219 (size=12001) 2024-12-02T05:17:47,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T05:17:47,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:47,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:47,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:47,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:47,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T05:17:47,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116727147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116727147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116727147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116727148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49568 deadline: 1733116727157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T05:17:47,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:47,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:47,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:47,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:47,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:47,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/26d3062617f74cc89f24d0dbcd4f163d 2024-12-02T05:17:47,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/5c59212df413455bb8958500cede782f is 50, key is test_row_0/C:col10/1733116666465/Put/seqid=0 2024-12-02T05:17:47,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T05:17:47,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:47,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:47,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:47,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:47,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742044_1220 (size=12001) 2024-12-02T05:17:47,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:47,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/5c59212df413455bb8958500cede782f 2024-12-02T05:17:47,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/6a65ffe569684ec48a15453f39d9bdfc as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6a65ffe569684ec48a15453f39d9bdfc 2024-12-02T05:17:47,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6a65ffe569684ec48a15453f39d9bdfc, entries=150, sequenceid=12, filesize=11.7 K 2024-12-02T05:17:47,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/26d3062617f74cc89f24d0dbcd4f163d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/26d3062617f74cc89f24d0dbcd4f163d 2024-12-02T05:17:47,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/26d3062617f74cc89f24d0dbcd4f163d, entries=150, sequenceid=12, filesize=11.7 K 2024-12-02T05:17:47,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/5c59212df413455bb8958500cede782f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5c59212df413455bb8958500cede782f 2024-12-02T05:17:47,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5c59212df413455bb8958500cede782f, entries=150, sequenceid=12, filesize=11.7 K 2024-12-02T05:17:47,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2bec4a5721d5a74dad6f846b3202c90f in 976ms, sequenceid=12, compaction requested=false 2024-12-02T05:17:47,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:47,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T05:17:47,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:47,536 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-02T05:17:47,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:47,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:47,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:47,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:47,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:47,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:47,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/57ac824115734051b601a32e449ac5a4 is 50, key is test_row_0/A:col10/1733116666515/Put/seqid=0 2024-12-02T05:17:47,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742045_1221 (size=12001) 2024-12-02T05:17:47,594 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/57ac824115734051b601a32e449ac5a4 2024-12-02T05:17:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T05:17:47,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/62a106b4afc641f78bade25c619d1202 is 50, key is test_row_0/B:col10/1733116666515/Put/seqid=0 2024-12-02T05:17:47,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742046_1222 (size=12001) 2024-12-02T05:17:47,639 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/62a106b4afc641f78bade25c619d1202 2024-12-02T05:17:47,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/821d8021ff4a4802a867dd4488fd53be is 50, key is test_row_0/C:col10/1733116666515/Put/seqid=0 2024-12-02T05:17:47,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742047_1223 (size=12001) 2024-12-02T05:17:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:47,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:47,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116727667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49568 deadline: 1733116727668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116727669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116727669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116727669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116727778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116727778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116727778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116727778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116727987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116727987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116727987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116727988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,053 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/821d8021ff4a4802a867dd4488fd53be 2024-12-02T05:17:48,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/57ac824115734051b601a32e449ac5a4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/57ac824115734051b601a32e449ac5a4 2024-12-02T05:17:48,061 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/57ac824115734051b601a32e449ac5a4, entries=150, sequenceid=48, filesize=11.7 K 2024-12-02T05:17:48,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/62a106b4afc641f78bade25c619d1202 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/62a106b4afc641f78bade25c619d1202 2024-12-02T05:17:48,078 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/62a106b4afc641f78bade25c619d1202, entries=150, sequenceid=48, filesize=11.7 K 2024-12-02T05:17:48,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/821d8021ff4a4802a867dd4488fd53be as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/821d8021ff4a4802a867dd4488fd53be 2024-12-02T05:17:48,083 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/821d8021ff4a4802a867dd4488fd53be, entries=150, sequenceid=48, filesize=11.7 K 2024-12-02T05:17:48,084 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 2bec4a5721d5a74dad6f846b3202c90f in 548ms, sequenceid=48, compaction requested=false 2024-12-02T05:17:48,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:48,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:48,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-02T05:17:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-02T05:17:48,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-02T05:17:48,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6290 sec 2024-12-02T05:17:48,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.6360 sec 2024-12-02T05:17:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:48,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T05:17:48,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:48,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:48,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:48,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:48,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:48,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:48,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/5334b8314c6042eaafc2d7df9c9a5582 is 50, key is test_row_0/A:col10/1733116667668/Put/seqid=0 2024-12-02T05:17:48,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742048_1224 (size=12001) 2024-12-02T05:17:48,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733116668299, max timestamp=1733116668322Num uniques 859; 2024-12-02T05:17:48,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T05:17:48,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116668308, max timestamp=1733116668322Num uniques 859; 2024-12-02T05:17:48,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:17:48,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116668308, max timestamp=1733116668322Num uniques 859; 2024-12-02T05:17:48,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:17:48,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116728339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116728340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116728343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116728344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116728448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116728448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116728457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116728457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T05:17:48,608 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:48,608 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-02T05:17:48,610 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:48,610 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:48,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T05:17:48,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116728657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116728657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116728667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116728667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49568 deadline: 1733116728688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,691 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in 2bec4a5721d5a74dad6f846b3202c90f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:48,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/5334b8314c6042eaafc2d7df9c9a5582 2024-12-02T05:17:48,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/1747502db267467d95056417b669a5e5 is 50, key is test_row_0/B:col10/1733116667668/Put/seqid=0 2024-12-02T05:17:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T05:17:48,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742049_1225 (size=12001) 2024-12-02T05:17:48,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T05:17:48,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:48,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:48,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:48,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:48,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:48,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T05:17:48,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:48,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:48,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:48,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:48,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T05:17:48,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116728967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116728967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116728977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116728978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:48,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:48,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:49,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T05:17:49,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:49,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:49,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:49,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:49,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:49,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:49,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/1747502db267467d95056417b669a5e5 2024-12-02T05:17:49,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/950968bbc92248e88c074e0b40762ec3 is 50, key is test_row_0/C:col10/1733116667668/Put/seqid=0 2024-12-02T05:17:49,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742050_1226 (size=12001) 2024-12-02T05:17:49,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/950968bbc92248e88c074e0b40762ec3 2024-12-02T05:17:49,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/5334b8314c6042eaafc2d7df9c9a5582 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/5334b8314c6042eaafc2d7df9c9a5582 2024-12-02T05:17:49,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/5334b8314c6042eaafc2d7df9c9a5582, entries=150, sequenceid=61, filesize=11.7 K 2024-12-02T05:17:49,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/1747502db267467d95056417b669a5e5 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/1747502db267467d95056417b669a5e5 2024-12-02T05:17:49,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/1747502db267467d95056417b669a5e5, entries=150, sequenceid=61, filesize=11.7 K 2024-12-02T05:17:49,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/950968bbc92248e88c074e0b40762ec3 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/950968bbc92248e88c074e0b40762ec3 2024-12-02T05:17:49,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/950968bbc92248e88c074e0b40762ec3, entries=150, sequenceid=61, filesize=11.7 K 2024-12-02T05:17:49,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=111.82 KB/114500 for 2bec4a5721d5a74dad6f846b3202c90f in 912ms, sequenceid=61, compaction requested=true 2024-12-02T05:17:49,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:49,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:17:49,211 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:49,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:49,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:17:49,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:49,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:17:49,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T05:17:49,211 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:49,212 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:49,212 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/A is initiating minor compaction (all files) 2024-12-02T05:17:49,213 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/A in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:49,213 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6a65ffe569684ec48a15453f39d9bdfc, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/57ac824115734051b601a32e449ac5a4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/5334b8314c6042eaafc2d7df9c9a5582] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=35.2 K 2024-12-02T05:17:49,213 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:49,213 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/B is initiating minor compaction (all files) 2024-12-02T05:17:49,213 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/B in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:49,213 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/26d3062617f74cc89f24d0dbcd4f163d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/62a106b4afc641f78bade25c619d1202, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/1747502db267467d95056417b669a5e5] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=35.2 K 2024-12-02T05:17:49,215 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a65ffe569684ec48a15453f39d9bdfc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733116666463 2024-12-02T05:17:49,215 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 26d3062617f74cc89f24d0dbcd4f163d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733116666463 2024-12-02T05:17:49,215 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57ac824115734051b601a32e449ac5a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733116666511 2024-12-02T05:17:49,215 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 62a106b4afc641f78bade25c619d1202, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733116666511 2024-12-02T05:17:49,216 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5334b8314c6042eaafc2d7df9c9a5582, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733116667668 2024-12-02T05:17:49,216 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 1747502db267467d95056417b669a5e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733116667668 2024-12-02T05:17:49,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T05:17:49,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:49,222 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=111.82 KB heapSize=315.45 KB 2024-12-02T05:17:49,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:49,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:49,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:49,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:49,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:49,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:49,234 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#A#compaction#185 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:49,235 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/e8be36c39a9842b792689401df308ac6 is 50, key is test_row_0/A:col10/1733116667668/Put/seqid=0 2024-12-02T05:17:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T05:17:49,242 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#B#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:49,243 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/b24942fbfe4b467a9bfa077d2c5c46ad is 50, key is test_row_0/B:col10/1733116667668/Put/seqid=0 2024-12-02T05:17:49,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/c6c32870794446a99f593b1c998fc0b4 is 50, key is test_row_0/A:col10/1733116668334/Put/seqid=0 2024-12-02T05:17:49,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742051_1227 (size=12104) 2024-12-02T05:17:49,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742053_1229 (size=12001) 2024-12-02T05:17:49,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742052_1228 (size=12104) 2024-12-02T05:17:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:49,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:49,481 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T05:17:49,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:49,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116729493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116729493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:49,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116729497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:49,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116729498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:49,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:49,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116729607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:49,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:49,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116729607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:49,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:49,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116729608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:49,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116729608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:49,681 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/c6c32870794446a99f593b1c998fc0b4 2024-12-02T05:17:49,688 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/b24942fbfe4b467a9bfa077d2c5c46ad as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/b24942fbfe4b467a9bfa077d2c5c46ad 2024-12-02T05:17:49,696 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/e8be36c39a9842b792689401df308ac6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/e8be36c39a9842b792689401df308ac6 2024-12-02T05:17:49,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/23d1bf3578bf4076991d0410e86eabe3 is 50, key is test_row_0/B:col10/1733116668334/Put/seqid=0 2024-12-02T05:17:49,704 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/B of 2bec4a5721d5a74dad6f846b3202c90f into b24942fbfe4b467a9bfa077d2c5c46ad(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:49,704 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:49,704 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/B, priority=13, startTime=1733116669211; duration=0sec 2024-12-02T05:17:49,704 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:49,704 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:B 2024-12-02T05:17:49,704 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:49,706 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:49,706 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/C is initiating minor compaction (all files) 2024-12-02T05:17:49,706 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/C in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:49,707 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5c59212df413455bb8958500cede782f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/821d8021ff4a4802a867dd4488fd53be, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/950968bbc92248e88c074e0b40762ec3] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=35.2 K 2024-12-02T05:17:49,707 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c59212df413455bb8958500cede782f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733116666463 2024-12-02T05:17:49,707 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 821d8021ff4a4802a867dd4488fd53be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733116666511 2024-12-02T05:17:49,708 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 950968bbc92248e88c074e0b40762ec3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733116667668 2024-12-02T05:17:49,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742054_1230 (size=12001) 2024-12-02T05:17:49,714 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/23d1bf3578bf4076991d0410e86eabe3 2024-12-02T05:17:49,714 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/A of 2bec4a5721d5a74dad6f846b3202c90f into e8be36c39a9842b792689401df308ac6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:49,714 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:49,714 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/A, priority=13, startTime=1733116669211; duration=0sec 2024-12-02T05:17:49,714 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:49,714 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:A 2024-12-02T05:17:49,719 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#C#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:49,719 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/c6dcd3ce68954736b2d25b6f070501f0 is 50, key is test_row_0/C:col10/1733116667668/Put/seqid=0 2024-12-02T05:17:49,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742055_1231 (size=12104) 2024-12-02T05:17:49,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/c9c32e33233c4615bb2c75786d9675f5 is 50, key is test_row_0/C:col10/1733116668334/Put/seqid=0 2024-12-02T05:17:49,734 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/c6dcd3ce68954736b2d25b6f070501f0 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/c6dcd3ce68954736b2d25b6f070501f0 2024-12-02T05:17:49,743 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/C of 2bec4a5721d5a74dad6f846b3202c90f into c6dcd3ce68954736b2d25b6f070501f0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:49,743 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:49,743 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/C, priority=13, startTime=1733116669211; duration=0sec 2024-12-02T05:17:49,743 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:49,743 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:C 2024-12-02T05:17:49,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742056_1232 (size=12001) 2024-12-02T05:17:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T05:17:49,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116729818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:49,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:49,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116729818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:49,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:49,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116729818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116729818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116730127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116730127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116730127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116730127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,148 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/c9c32e33233c4615bb2c75786d9675f5 2024-12-02T05:17:50,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/c6c32870794446a99f593b1c998fc0b4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c6c32870794446a99f593b1c998fc0b4 2024-12-02T05:17:50,158 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c6c32870794446a99f593b1c998fc0b4, entries=150, sequenceid=94, filesize=11.7 K 2024-12-02T05:17:50,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/23d1bf3578bf4076991d0410e86eabe3 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/23d1bf3578bf4076991d0410e86eabe3 2024-12-02T05:17:50,164 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/23d1bf3578bf4076991d0410e86eabe3, entries=150, sequenceid=94, filesize=11.7 K 2024-12-02T05:17:50,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/c9c32e33233c4615bb2c75786d9675f5 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/c9c32e33233c4615bb2c75786d9675f5 2024-12-02T05:17:50,170 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/c9c32e33233c4615bb2c75786d9675f5, entries=150, sequenceid=94, filesize=11.7 K 2024-12-02T05:17:50,172 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~111.82 KB/114500, heapSize ~315.41 KB/322976, currentSize=67.09 KB/68700 for 2bec4a5721d5a74dad6f846b3202c90f in 950ms, sequenceid=94, compaction requested=false 2024-12-02T05:17:50,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:50,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:50,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-02T05:17:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-02T05:17:50,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-02T05:17:50,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5640 sec 2024-12-02T05:17:50,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.5710 sec 2024-12-02T05:17:50,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T05:17:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:50,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:50,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:50,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:50,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:50,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:50,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:50,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/7ed3970cb2424a3582b124cbabe4ca64 is 50, key is test_row_0/A:col10/1733116670640/Put/seqid=0 2024-12-02T05:17:50,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742057_1233 (size=14341) 2024-12-02T05:17:50,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116730664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116730664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116730664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116730665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49568 deadline: 1733116730718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,719 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in 2bec4a5721d5a74dad6f846b3202c90f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:50,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T05:17:50,757 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:50,758 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-02T05:17:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T05:17:50,760 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:50,761 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:50,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:50,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116730777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116730777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116730778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116730778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T05:17:50,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T05:17:50,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:50,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:50,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:50,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:50,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:50,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116730988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116730988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116730988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116730988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:51,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/7ed3970cb2424a3582b124cbabe4ca64 2024-12-02T05:17:51,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T05:17:51,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:51,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:51,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:51,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:51,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:51,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/c9180f7c9af04f1f90b6061eac7da8b7 is 50, key is test_row_0/B:col10/1733116670640/Put/seqid=0 2024-12-02T05:17:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T05:17:51,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742058_1234 (size=12001) 2024-12-02T05:17:51,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/c9180f7c9af04f1f90b6061eac7da8b7 2024-12-02T05:17:51,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/7b042a07248d43bfbf94b9331ffb41ed is 50, key is test_row_0/C:col10/1733116670640/Put/seqid=0 2024-12-02T05:17:51,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742059_1235 (size=12001) 2024-12-02T05:17:51,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T05:17:51,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:51,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:51,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:51,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:51,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:51,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116731297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:51,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116731297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:51,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116731297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116731297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:51,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T05:17:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:51,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:51,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T05:17:51,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/7b042a07248d43bfbf94b9331ffb41ed 2024-12-02T05:17:51,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/7ed3970cb2424a3582b124cbabe4ca64 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/7ed3970cb2424a3582b124cbabe4ca64 2024-12-02T05:17:51,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/7ed3970cb2424a3582b124cbabe4ca64, entries=200, sequenceid=114, filesize=14.0 K 2024-12-02T05:17:51,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/c9180f7c9af04f1f90b6061eac7da8b7 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/c9180f7c9af04f1f90b6061eac7da8b7 2024-12-02T05:17:51,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/c9180f7c9af04f1f90b6061eac7da8b7, entries=150, sequenceid=114, filesize=11.7 K 2024-12-02T05:17:51,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/7b042a07248d43bfbf94b9331ffb41ed as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/7b042a07248d43bfbf94b9331ffb41ed 2024-12-02T05:17:51,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/7b042a07248d43bfbf94b9331ffb41ed, entries=150, sequenceid=114, filesize=11.7 K 2024-12-02T05:17:51,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 2bec4a5721d5a74dad6f846b3202c90f in 884ms, sequenceid=114, compaction requested=true 2024-12-02T05:17:51,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:51,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:17:51,525 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:51,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:51,526 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:51,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:17:51,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:51,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:17:51,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:51,527 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:51,527 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:51,527 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/B is initiating minor compaction (all files) 2024-12-02T05:17:51,527 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/A is initiating minor compaction (all files) 2024-12-02T05:17:51,527 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/B in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:51,527 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/A in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:51,527 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/e8be36c39a9842b792689401df308ac6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c6c32870794446a99f593b1c998fc0b4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/7ed3970cb2424a3582b124cbabe4ca64] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=37.5 K 2024-12-02T05:17:51,527 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/b24942fbfe4b467a9bfa077d2c5c46ad, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/23d1bf3578bf4076991d0410e86eabe3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/c9180f7c9af04f1f90b6061eac7da8b7] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=35.3 K 2024-12-02T05:17:51,528 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8be36c39a9842b792689401df308ac6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733116667668 2024-12-02T05:17:51,528 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting b24942fbfe4b467a9bfa077d2c5c46ad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733116667668 2024-12-02T05:17:51,528 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6c32870794446a99f593b1c998fc0b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733116668334 2024-12-02T05:17:51,528 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 23d1bf3578bf4076991d0410e86eabe3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733116668334 2024-12-02T05:17:51,529 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ed3970cb2424a3582b124cbabe4ca64, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733116669490 2024-12-02T05:17:51,529 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting c9180f7c9af04f1f90b6061eac7da8b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733116669490 2024-12-02T05:17:51,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T05:17:51,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:51,529 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T05:17:51,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:51,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:51,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:51,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:51,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:51,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:51,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/271dbbf536724f1fa3dc4be16d5eba1b is 50, key is test_row_0/A:col10/1733116670662/Put/seqid=0 2024-12-02T05:17:51,558 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#B#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:51,558 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#A#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:51,558 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/c744f96702af428e81df8bd0f3eb65fa is 50, key is test_row_0/A:col10/1733116670640/Put/seqid=0 2024-12-02T05:17:51,558 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/3ad5dc984cb94c11ae62471864c21a64 is 50, key is test_row_0/B:col10/1733116670640/Put/seqid=0 2024-12-02T05:17:51,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742060_1236 (size=12001) 2024-12-02T05:17:51,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742062_1238 (size=12207) 2024-12-02T05:17:51,594 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/271dbbf536724f1fa3dc4be16d5eba1b 2024-12-02T05:17:51,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742061_1237 (size=12207) 2024-12-02T05:17:51,601 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/3ad5dc984cb94c11ae62471864c21a64 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/3ad5dc984cb94c11ae62471864c21a64 2024-12-02T05:17:51,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/628f0cdd383041f091f7ca8c3865f6f7 is 50, key is test_row_0/B:col10/1733116670662/Put/seqid=0 2024-12-02T05:17:51,607 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/c744f96702af428e81df8bd0f3eb65fa as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c744f96702af428e81df8bd0f3eb65fa 2024-12-02T05:17:51,609 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/B of 2bec4a5721d5a74dad6f846b3202c90f into 3ad5dc984cb94c11ae62471864c21a64(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:51,609 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:51,609 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/B, priority=13, startTime=1733116671525; duration=0sec 2024-12-02T05:17:51,609 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:51,609 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:B 2024-12-02T05:17:51,609 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:51,610 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:51,611 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/C is initiating minor compaction (all files) 2024-12-02T05:17:51,611 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/C in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:51,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742063_1239 (size=12001) 2024-12-02T05:17:51,611 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/c6dcd3ce68954736b2d25b6f070501f0, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/c9c32e33233c4615bb2c75786d9675f5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/7b042a07248d43bfbf94b9331ffb41ed] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=35.3 K 2024-12-02T05:17:51,612 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/628f0cdd383041f091f7ca8c3865f6f7 2024-12-02T05:17:51,612 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting c6dcd3ce68954736b2d25b6f070501f0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733116667668 2024-12-02T05:17:51,613 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting c9c32e33233c4615bb2c75786d9675f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733116668334 2024-12-02T05:17:51,613 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b042a07248d43bfbf94b9331ffb41ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733116669490 2024-12-02T05:17:51,613 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/A of 2bec4a5721d5a74dad6f846b3202c90f into c744f96702af428e81df8bd0f3eb65fa(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:51,614 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:51,614 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/A, priority=13, startTime=1733116671525; duration=0sec 2024-12-02T05:17:51,614 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:51,614 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:A 2024-12-02T05:17:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/b0203495d92a4610b9af71c56f64dfce is 50, key is test_row_0/C:col10/1733116670662/Put/seqid=0 2024-12-02T05:17:51,621 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#C#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:51,622 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/8dd39b7153f344778ecd463098b45d6a is 50, key is test_row_0/C:col10/1733116670640/Put/seqid=0 2024-12-02T05:17:51,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742064_1240 (size=12001) 2024-12-02T05:17:51,633 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/b0203495d92a4610b9af71c56f64dfce 2024-12-02T05:17:51,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742065_1241 (size=12207) 2024-12-02T05:17:51,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/271dbbf536724f1fa3dc4be16d5eba1b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/271dbbf536724f1fa3dc4be16d5eba1b 2024-12-02T05:17:51,644 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/271dbbf536724f1fa3dc4be16d5eba1b, entries=150, sequenceid=129, filesize=11.7 K 2024-12-02T05:17:51,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/628f0cdd383041f091f7ca8c3865f6f7 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/628f0cdd383041f091f7ca8c3865f6f7 2024-12-02T05:17:51,653 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/628f0cdd383041f091f7ca8c3865f6f7, entries=150, sequenceid=129, filesize=11.7 K 2024-12-02T05:17:51,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/b0203495d92a4610b9af71c56f64dfce as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/b0203495d92a4610b9af71c56f64dfce 2024-12-02T05:17:51,669 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/b0203495d92a4610b9af71c56f64dfce, entries=150, sequenceid=129, filesize=11.7 K 2024-12-02T05:17:51,671 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 2bec4a5721d5a74dad6f846b3202c90f in 141ms, sequenceid=129, compaction requested=false 2024-12-02T05:17:51,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:51,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:51,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-02T05:17:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-02T05:17:51,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-02T05:17:51,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 912 msec 2024-12-02T05:17:51,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 920 msec 2024-12-02T05:17:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:51,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T05:17:51,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:51,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:51,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:51,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:51,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:51,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:51,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/2abd8aefbb0b4797af2d507ec03b5a7d is 50, key is test_row_0/A:col10/1733116671811/Put/seqid=0 2024-12-02T05:17:51,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742066_1242 (size=12151) 2024-12-02T05:17:51,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/2abd8aefbb0b4797af2d507ec03b5a7d 2024-12-02T05:17:51,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/870e8e5d496647af912c917415f3c29c is 50, key is test_row_0/B:col10/1733116671811/Put/seqid=0 2024-12-02T05:17:51,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742067_1243 (size=12151) 2024-12-02T05:17:51,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:51,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733116671831, max timestamp=1733116671849Num uniques 859; 2024-12-02T05:17:51,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733116671831, max timestamp=1733116671849Num uniques 859; 2024-12-02T05:17:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116731845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:51,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T05:17:51,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:17:51,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733116671831, max timestamp=1733116671849Num uniques 859; 2024-12-02T05:17:51,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:17:51,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T05:17:51,897 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:51,898 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-02T05:17:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T05:17:51,901 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:51,902 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:51,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:51,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116731897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:51,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116731897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:51,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116731900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:51,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116731978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T05:17:52,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116732018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:52,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116732018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:52,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:52,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116732018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:52,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:52,040 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/8dd39b7153f344778ecd463098b45d6a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/8dd39b7153f344778ecd463098b45d6a 2024-12-02T05:17:52,045 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/C of 2bec4a5721d5a74dad6f846b3202c90f into 8dd39b7153f344778ecd463098b45d6a(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:52,045 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:52,045 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/C, priority=13, startTime=1733116671526; duration=0sec 2024-12-02T05:17:52,045 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:52,045 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:C 2024-12-02T05:17:52,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T05:17:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:52,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:52,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:52,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:52,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116732188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:52,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T05:17:52,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:52,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:52,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:52,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:52,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T05:17:52,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116732227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:52,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:52,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116732227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:52,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116732227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:52,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/870e8e5d496647af912c917415f3c29c 2024-12-02T05:17:52,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/438335314d744dd38a3060f0674c7d9e is 50, key is test_row_0/C:col10/1733116671811/Put/seqid=0 2024-12-02T05:17:52,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742068_1244 (size=12151) 2024-12-02T05:17:52,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T05:17:52,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:52,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:52,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:52,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116732497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:52,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T05:17:52,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:52,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:52,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:52,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:52,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T05:17:52,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116732537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:52,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:52,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116732537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:52,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116732537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:52,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T05:17:52,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:52,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:52,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:52,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:52,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:52,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/438335314d744dd38a3060f0674c7d9e 2024-12-02T05:17:52,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/2abd8aefbb0b4797af2d507ec03b5a7d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/2abd8aefbb0b4797af2d507ec03b5a7d 2024-12-02T05:17:52,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/2abd8aefbb0b4797af2d507ec03b5a7d, entries=150, sequenceid=142, filesize=11.9 K 2024-12-02T05:17:52,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/870e8e5d496647af912c917415f3c29c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/870e8e5d496647af912c917415f3c29c 2024-12-02T05:17:52,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/870e8e5d496647af912c917415f3c29c, entries=150, sequenceid=142, filesize=11.9 K 2024-12-02T05:17:52,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/438335314d744dd38a3060f0674c7d9e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/438335314d744dd38a3060f0674c7d9e 2024-12-02T05:17:52,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/438335314d744dd38a3060f0674c7d9e, entries=150, sequenceid=142, filesize=11.9 K 2024-12-02T05:17:52,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.42 KB/139690 for 2bec4a5721d5a74dad6f846b3202c90f in 877ms, sequenceid=142, compaction requested=true 2024-12-02T05:17:52,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:52,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:17:52,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:52,691 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:52,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:17:52,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:52,691 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:52,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:17:52,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:52,692 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:52,692 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/B is initiating minor compaction (all files) 2024-12-02T05:17:52,692 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/B in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:52,693 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/3ad5dc984cb94c11ae62471864c21a64, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/628f0cdd383041f091f7ca8c3865f6f7, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/870e8e5d496647af912c917415f3c29c] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=35.5 K 2024-12-02T05:17:52,693 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:52,693 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/A is initiating minor compaction (all files) 2024-12-02T05:17:52,693 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/A in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:52,693 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c744f96702af428e81df8bd0f3eb65fa, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/271dbbf536724f1fa3dc4be16d5eba1b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/2abd8aefbb0b4797af2d507ec03b5a7d] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=35.5 K 2024-12-02T05:17:52,694 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ad5dc984cb94c11ae62471864c21a64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733116669490 2024-12-02T05:17:52,694 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting c744f96702af428e81df8bd0f3eb65fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733116669490 2024-12-02T05:17:52,694 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 271dbbf536724f1fa3dc4be16d5eba1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733116670662 2024-12-02T05:17:52,694 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 628f0cdd383041f091f7ca8c3865f6f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733116670662 2024-12-02T05:17:52,694 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2abd8aefbb0b4797af2d507ec03b5a7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733116671811 2024-12-02T05:17:52,694 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 870e8e5d496647af912c917415f3c29c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733116671811 2024-12-02T05:17:52,702 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#B#compaction#203 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:52,703 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/652ac67b2b3e41b0979a72862c55c493 is 50, key is test_row_0/B:col10/1733116671811/Put/seqid=0 2024-12-02T05:17:52,722 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#A#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:52,723 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/749850001c464067b51264445456d464 is 50, key is test_row_0/A:col10/1733116671811/Put/seqid=0 2024-12-02T05:17:52,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742069_1245 (size=12459) 2024-12-02T05:17:52,759 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/652ac67b2b3e41b0979a72862c55c493 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/652ac67b2b3e41b0979a72862c55c493 2024-12-02T05:17:52,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742070_1246 (size=12459) 2024-12-02T05:17:52,768 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/B of 2bec4a5721d5a74dad6f846b3202c90f into 652ac67b2b3e41b0979a72862c55c493(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:52,768 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/749850001c464067b51264445456d464 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/749850001c464067b51264445456d464 2024-12-02T05:17:52,768 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:52,768 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/B, priority=13, startTime=1733116672691; duration=0sec 2024-12-02T05:17:52,768 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:52,768 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:B 2024-12-02T05:17:52,768 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:52,770 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:52,770 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/C is initiating minor compaction (all files) 2024-12-02T05:17:52,770 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/C in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:52,770 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/8dd39b7153f344778ecd463098b45d6a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/b0203495d92a4610b9af71c56f64dfce, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/438335314d744dd38a3060f0674c7d9e] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=35.5 K 2024-12-02T05:17:52,771 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dd39b7153f344778ecd463098b45d6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733116669490 2024-12-02T05:17:52,772 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting b0203495d92a4610b9af71c56f64dfce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733116670662 2024-12-02T05:17:52,772 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 438335314d744dd38a3060f0674c7d9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733116671811 2024-12-02T05:17:52,784 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/A of 2bec4a5721d5a74dad6f846b3202c90f into 749850001c464067b51264445456d464(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:52,784 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:52,785 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/A, priority=13, startTime=1733116672691; duration=0sec 2024-12-02T05:17:52,786 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:52,786 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:A 2024-12-02T05:17:52,810 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#C#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:52,812 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/d04e18e9ce2c409d85a30b3d3a2178cd is 50, key is test_row_0/C:col10/1733116671811/Put/seqid=0 2024-12-02T05:17:52,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T05:17:52,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:52,820 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-12-02T05:17:52,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:52,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:52,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:52,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:52,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:52,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:52,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742071_1247 (size=12459) 2024-12-02T05:17:52,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/97d5ec58a75b45ba999d52e5a77790a3 is 50, key is test_row_0/A:col10/1733116671897/Put/seqid=0 2024-12-02T05:17:52,848 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/d04e18e9ce2c409d85a30b3d3a2178cd as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/d04e18e9ce2c409d85a30b3d3a2178cd 2024-12-02T05:17:52,855 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/C of 2bec4a5721d5a74dad6f846b3202c90f into d04e18e9ce2c409d85a30b3d3a2178cd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:52,855 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:52,855 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/C, priority=13, startTime=1733116672691; duration=0sec 2024-12-02T05:17:52,856 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:52,856 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:C 2024-12-02T05:17:52,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742072_1248 (size=12151) 2024-12-02T05:17:52,877 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/97d5ec58a75b45ba999d52e5a77790a3 2024-12-02T05:17:52,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/d5bdc84e30034c6b9ff0224cb0fab444 is 50, key is test_row_0/B:col10/1733116671897/Put/seqid=0 2024-12-02T05:17:52,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742073_1249 (size=12151) 2024-12-02T05:17:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:53,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:53,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116733034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T05:17:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:53,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:53,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:53,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116733047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:53,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116733047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:53,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:53,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116733047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:53,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:53,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:53,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:53,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116733147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:53,310 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/d5bdc84e30034c6b9ff0224cb0fab444 2024-12-02T05:17:53,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/5f9121cfd9654631a33aa5021e9441cd is 50, key is test_row_0/C:col10/1733116671897/Put/seqid=0 2024-12-02T05:17:53,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742074_1250 (size=12151) 2024-12-02T05:17:53,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116733357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:53,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:53,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:53,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:53,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:53,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116733667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:53,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:53,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:53,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:53,731 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/5f9121cfd9654631a33aa5021e9441cd 2024-12-02T05:17:53,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/97d5ec58a75b45ba999d52e5a77790a3 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/97d5ec58a75b45ba999d52e5a77790a3 2024-12-02T05:17:53,740 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/97d5ec58a75b45ba999d52e5a77790a3, entries=150, sequenceid=181, filesize=11.9 K 2024-12-02T05:17:53,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/d5bdc84e30034c6b9ff0224cb0fab444 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/d5bdc84e30034c6b9ff0224cb0fab444 2024-12-02T05:17:53,745 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/d5bdc84e30034c6b9ff0224cb0fab444, entries=150, sequenceid=181, filesize=11.9 K 2024-12-02T05:17:53,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/5f9121cfd9654631a33aa5021e9441cd as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5f9121cfd9654631a33aa5021e9441cd 2024-12-02T05:17:53,749 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5f9121cfd9654631a33aa5021e9441cd, entries=150, sequenceid=181, filesize=11.9 K 2024-12-02T05:17:53,750 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=46.96 KB/48090 for 2bec4a5721d5a74dad6f846b3202c90f in 930ms, sequenceid=181, compaction requested=false 2024-12-02T05:17:53,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:53,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:53,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-02T05:17:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-02T05:17:53,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-02T05:17:53,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8490 sec 2024-12-02T05:17:53,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.8550 sec 2024-12-02T05:17:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T05:17:54,047 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:54,048 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-02T05:17:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T05:17:54,049 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:54,052 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:54,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:54,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T05:17:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:54,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/55649d932d3f44c2852953ab121e4315 is 50, key is test_row_0/A:col10/1733116673032/Put/seqid=0 2024-12-02T05:17:54,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742075_1251 (size=14541) 2024-12-02T05:17:54,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116734138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:54,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116734138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:54,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116734141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T05:17:54,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116734177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:54,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T05:17:54,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:54,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:54,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:54,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:54,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:54,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116734247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:54,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116734247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116734248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:54,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T05:17:54,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:54,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:54,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:54,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:54,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T05:17:54,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116734458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:54,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:54,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:54,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116734458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116734458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:54,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/55649d932d3f44c2852953ab121e4315 2024-12-02T05:17:54,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/e5c35caa873c452caabcd1d99c11db68 is 50, key is test_row_0/B:col10/1733116673032/Put/seqid=0 2024-12-02T05:17:54,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742076_1252 (size=12151) 2024-12-02T05:17:54,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T05:17:54,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:54,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:54,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:54,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:54,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:54,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T05:17:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:54,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T05:17:54,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49568 deadline: 1733116734748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:54,749 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in 2bec4a5721d5a74dad6f846b3202c90f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:17:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:54,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116734767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:54,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116734767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116734767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:54,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T05:17:54,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:54,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:54,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:54,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:54,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:54,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/e5c35caa873c452caabcd1d99c11db68 2024-12-02T05:17:54,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/aeba02159c144544b646e843fbd3a9cd is 50, key is test_row_0/C:col10/1733116673032/Put/seqid=0 2024-12-02T05:17:54,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742077_1253 (size=12151) 2024-12-02T05:17:54,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/aeba02159c144544b646e843fbd3a9cd 2024-12-02T05:17:54,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/55649d932d3f44c2852953ab121e4315 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/55649d932d3f44c2852953ab121e4315 2024-12-02T05:17:54,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/55649d932d3f44c2852953ab121e4315, entries=200, sequenceid=194, filesize=14.2 K 2024-12-02T05:17:54,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/e5c35caa873c452caabcd1d99c11db68 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/e5c35caa873c452caabcd1d99c11db68 2024-12-02T05:17:54,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/e5c35caa873c452caabcd1d99c11db68, entries=150, sequenceid=194, filesize=11.9 K 2024-12-02T05:17:54,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/aeba02159c144544b646e843fbd3a9cd as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/aeba02159c144544b646e843fbd3a9cd 2024-12-02T05:17:54,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/aeba02159c144544b646e843fbd3a9cd, entries=150, sequenceid=194, filesize=11.9 K 2024-12-02T05:17:54,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2bec4a5721d5a74dad6f846b3202c90f in 877ms, sequenceid=194, compaction requested=true 2024-12-02T05:17:54,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:54,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:17:54,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:54,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:17:54,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:54,936 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:54,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:17:54,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:54,936 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:54,937 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:54,937 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/A is initiating minor compaction (all files) 2024-12-02T05:17:54,937 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/A in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:54,938 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/749850001c464067b51264445456d464, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/97d5ec58a75b45ba999d52e5a77790a3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/55649d932d3f44c2852953ab121e4315] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=38.2 K 2024-12-02T05:17:54,938 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:54,938 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/B is initiating minor compaction (all files) 2024-12-02T05:17:54,938 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/B in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:54,938 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/652ac67b2b3e41b0979a72862c55c493, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/d5bdc84e30034c6b9ff0224cb0fab444, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/e5c35caa873c452caabcd1d99c11db68] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=35.9 K 2024-12-02T05:17:54,939 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 749850001c464067b51264445456d464, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733116671811 2024-12-02T05:17:54,939 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 652ac67b2b3e41b0979a72862c55c493, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733116671811 2024-12-02T05:17:54,940 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 97d5ec58a75b45ba999d52e5a77790a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733116671883 2024-12-02T05:17:54,940 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5bdc84e30034c6b9ff0224cb0fab444, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733116671883 2024-12-02T05:17:54,940 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5c35caa873c452caabcd1d99c11db68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733116673011 2024-12-02T05:17:54,941 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 55649d932d3f44c2852953ab121e4315, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733116673011 2024-12-02T05:17:54,949 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#B#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:54,949 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/b8066ac2975c42f7aa9aff53da440d84 is 50, key is test_row_0/B:col10/1733116673032/Put/seqid=0 2024-12-02T05:17:54,950 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:54,950 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/6587565a6b584ec3befdd2d36b2b8987 is 50, key is test_row_0/A:col10/1733116673032/Put/seqid=0 2024-12-02T05:17:54,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T05:17:54,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:54,971 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T05:17:54,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:54,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:54,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:54,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:54,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:54,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:54,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742079_1255 (size=12561) 2024-12-02T05:17:54,981 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/6587565a6b584ec3befdd2d36b2b8987 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6587565a6b584ec3befdd2d36b2b8987 2024-12-02T05:17:54,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/8861f85a53e945738817850eb023eaa1 is 50, key is test_row_0/A:col10/1733116674140/Put/seqid=0 2024-12-02T05:17:54,998 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/A of 2bec4a5721d5a74dad6f846b3202c90f into 6587565a6b584ec3befdd2d36b2b8987(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:54,998 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:54,998 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/A, priority=13, startTime=1733116674936; duration=0sec 2024-12-02T05:17:54,998 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:54,998 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:A 2024-12-02T05:17:54,998 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:55,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742078_1254 (size=12561) 2024-12-02T05:17:55,002 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:55,002 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/C is initiating minor compaction (all files) 2024-12-02T05:17:55,002 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/C in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:55,002 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/d04e18e9ce2c409d85a30b3d3a2178cd, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5f9121cfd9654631a33aa5021e9441cd, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/aeba02159c144544b646e843fbd3a9cd] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=35.9 K 2024-12-02T05:17:55,011 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting d04e18e9ce2c409d85a30b3d3a2178cd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733116671811 2024-12-02T05:17:55,011 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f9121cfd9654631a33aa5021e9441cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733116671883 2024-12-02T05:17:55,017 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting aeba02159c144544b646e843fbd3a9cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733116673011 2024-12-02T05:17:55,019 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/b8066ac2975c42f7aa9aff53da440d84 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/b8066ac2975c42f7aa9aff53da440d84 2024-12-02T05:17:55,030 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/B of 2bec4a5721d5a74dad6f846b3202c90f into b8066ac2975c42f7aa9aff53da440d84(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:55,030 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:55,030 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/B, priority=13, startTime=1733116674936; duration=0sec 2024-12-02T05:17:55,030 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:55,030 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:B 2024-12-02T05:17:55,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742080_1256 (size=12151) 2024-12-02T05:17:55,034 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/8861f85a53e945738817850eb023eaa1 2024-12-02T05:17:55,038 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#C#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:55,038 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/09c1d122f31a4673a8fae16b5e9e0eca is 50, key is test_row_0/C:col10/1733116673032/Put/seqid=0 2024-12-02T05:17:55,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/0ed20baf1872444a8de5061b3592b9f4 is 50, key is test_row_0/B:col10/1733116674140/Put/seqid=0 2024-12-02T05:17:55,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742081_1257 (size=12561) 2024-12-02T05:17:55,110 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/09c1d122f31a4673a8fae16b5e9e0eca as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/09c1d122f31a4673a8fae16b5e9e0eca 2024-12-02T05:17:55,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742082_1258 (size=12151) 2024-12-02T05:17:55,115 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/0ed20baf1872444a8de5061b3592b9f4 2024-12-02T05:17:55,117 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/C of 2bec4a5721d5a74dad6f846b3202c90f into 09c1d122f31a4673a8fae16b5e9e0eca(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:55,117 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:55,117 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/C, priority=13, startTime=1733116674936; duration=0sec 2024-12-02T05:17:55,118 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:55,118 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:C 2024-12-02T05:17:55,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/367be8ae7812407c814c9594881035f2 is 50, key is test_row_0/C:col10/1733116674140/Put/seqid=0 2024-12-02T05:17:55,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742083_1259 (size=12151) 2024-12-02T05:17:55,149 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/367be8ae7812407c814c9594881035f2 2024-12-02T05:17:55,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/8861f85a53e945738817850eb023eaa1 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/8861f85a53e945738817850eb023eaa1 2024-12-02T05:17:55,169 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/8861f85a53e945738817850eb023eaa1, entries=150, sequenceid=214, filesize=11.9 K 2024-12-02T05:17:55,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/0ed20baf1872444a8de5061b3592b9f4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/0ed20baf1872444a8de5061b3592b9f4 2024-12-02T05:17:55,185 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/0ed20baf1872444a8de5061b3592b9f4, entries=150, sequenceid=214, filesize=11.9 K 2024-12-02T05:17:55,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/367be8ae7812407c814c9594881035f2 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/367be8ae7812407c814c9594881035f2 2024-12-02T05:17:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T05:17:55,192 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/367be8ae7812407c814c9594881035f2, entries=150, sequenceid=214, filesize=11.9 K 2024-12-02T05:17:55,193 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 2bec4a5721d5a74dad6f846b3202c90f in 222ms, sequenceid=214, compaction requested=false 2024-12-02T05:17:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-02T05:17:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-02T05:17:55,197 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-02T05:17:55,197 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1430 sec 2024-12-02T05:17:55,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.1500 sec 2024-12-02T05:17:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:55,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T05:17:55,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:55,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:55,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:55,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:55,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:55,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:55,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/798c3a63ab5a46f99601cc4ae18a4357 is 50, key is test_row_0/A:col10/1733116675210/Put/seqid=0 2024-12-02T05:17:55,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742084_1260 (size=14541) 2024-12-02T05:17:55,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/798c3a63ab5a46f99601cc4ae18a4357 2024-12-02T05:17:55,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116675276, max timestamp=1733116675291Num uniques 859; 2024-12-02T05:17:55,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116675276, max timestamp=1733116675291Num uniques 859; 2024-12-02T05:17:55,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:17:55,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:17:55,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/8989b1e9e5594fe886bb3e72c557741a is 50, key is test_row_0/B:col10/1733116675210/Put/seqid=0 2024-12-02T05:17:55,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116675276, max timestamp=1733116675291Num uniques 859; 2024-12-02T05:17:55,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:17:55,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742085_1261 (size=12151) 2024-12-02T05:17:55,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/8989b1e9e5594fe886bb3e72c557741a 2024-12-02T05:17:55,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116735318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116735319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:55,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:55,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116735320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:55,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116735321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/a3581974ef1a427f90e086feb86d4f41 is 50, key is test_row_0/C:col10/1733116675210/Put/seqid=0 2024-12-02T05:17:55,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:55,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742086_1262 (size=12151) 2024-12-02T05:17:55,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/a3581974ef1a427f90e086feb86d4f41 2024-12-02T05:17:55,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/798c3a63ab5a46f99601cc4ae18a4357 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/798c3a63ab5a46f99601cc4ae18a4357 2024-12-02T05:17:55,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/798c3a63ab5a46f99601cc4ae18a4357, entries=200, sequenceid=227, filesize=14.2 K 2024-12-02T05:17:55,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/8989b1e9e5594fe886bb3e72c557741a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/8989b1e9e5594fe886bb3e72c557741a 2024-12-02T05:17:55,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/8989b1e9e5594fe886bb3e72c557741a, entries=150, sequenceid=227, filesize=11.9 K 2024-12-02T05:17:55,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/a3581974ef1a427f90e086feb86d4f41 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/a3581974ef1a427f90e086feb86d4f41 2024-12-02T05:17:55,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/a3581974ef1a427f90e086feb86d4f41, entries=150, sequenceid=227, filesize=11.9 K 2024-12-02T05:17:55,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 2bec4a5721d5a74dad6f846b3202c90f in 166ms, sequenceid=227, compaction requested=true 2024-12-02T05:17:55,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:55,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:17:55,386 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:55,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:55,386 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:55,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:17:55,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:55,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:17:55,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:55,388 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:55,388 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:55,388 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/B is initiating minor compaction (all files) 2024-12-02T05:17:55,388 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/A is initiating minor compaction (all files) 2024-12-02T05:17:55,388 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/B in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:55,388 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/A in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:55,388 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6587565a6b584ec3befdd2d36b2b8987, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/8861f85a53e945738817850eb023eaa1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/798c3a63ab5a46f99601cc4ae18a4357] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=38.3 K 2024-12-02T05:17:55,388 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/b8066ac2975c42f7aa9aff53da440d84, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/0ed20baf1872444a8de5061b3592b9f4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/8989b1e9e5594fe886bb3e72c557741a] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=36.0 K 2024-12-02T05:17:55,394 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting b8066ac2975c42f7aa9aff53da440d84, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733116673011 2024-12-02T05:17:55,394 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6587565a6b584ec3befdd2d36b2b8987, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733116673011 2024-12-02T05:17:55,395 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ed20baf1872444a8de5061b3592b9f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733116674074 2024-12-02T05:17:55,395 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8861f85a53e945738817850eb023eaa1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733116674074 2024-12-02T05:17:55,395 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 798c3a63ab5a46f99601cc4ae18a4357, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733116675206 2024-12-02T05:17:55,395 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 8989b1e9e5594fe886bb3e72c557741a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733116675206 2024-12-02T05:17:55,414 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#A#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:55,415 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/aa6dfd7e501046d2bface455a06a89a6 is 50, key is test_row_0/A:col10/1733116675210/Put/seqid=0 2024-12-02T05:17:55,418 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#B#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:55,418 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/a8ce8eef8fb149d08b433d011db86865 is 50, key is test_row_0/B:col10/1733116675210/Put/seqid=0 2024-12-02T05:17:55,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742087_1263 (size=12663) 2024-12-02T05:17:55,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742088_1264 (size=12663) 2024-12-02T05:17:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:55,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-02T05:17:55,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:55,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:55,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:55,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:55,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:55,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:55,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/4c56febae6384a1ba1158e23892c086e is 50, key is test_row_0/A:col10/1733116675318/Put/seqid=0 2024-12-02T05:17:55,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116735436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742089_1265 (size=14741) 2024-12-02T05:17:55,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116735437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116735437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/4c56febae6384a1ba1158e23892c086e 2024-12-02T05:17:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:55,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116735441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:55,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/0039dedad5f3473fb9488d0de2979c9e is 50, key is test_row_0/B:col10/1733116675318/Put/seqid=0 2024-12-02T05:17:55,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742090_1266 (size=12301) 2024-12-02T05:17:55,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116735548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116735548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:55,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116735557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:55,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116735647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116735758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116735758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:55,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116735767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:55,826 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/aa6dfd7e501046d2bface455a06a89a6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/aa6dfd7e501046d2bface455a06a89a6 2024-12-02T05:17:55,833 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/a8ce8eef8fb149d08b433d011db86865 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/a8ce8eef8fb149d08b433d011db86865 2024-12-02T05:17:55,833 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/A of 2bec4a5721d5a74dad6f846b3202c90f into aa6dfd7e501046d2bface455a06a89a6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:55,833 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:55,833 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/A, priority=13, startTime=1733116675386; duration=0sec 2024-12-02T05:17:55,833 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:55,833 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:A 2024-12-02T05:17:55,833 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:55,834 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:55,834 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/C is initiating minor compaction (all files) 2024-12-02T05:17:55,834 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/C in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:55,834 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/09c1d122f31a4673a8fae16b5e9e0eca, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/367be8ae7812407c814c9594881035f2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/a3581974ef1a427f90e086feb86d4f41] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=36.0 K 2024-12-02T05:17:55,834 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09c1d122f31a4673a8fae16b5e9e0eca, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733116673011 2024-12-02T05:17:55,835 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 367be8ae7812407c814c9594881035f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733116674074 2024-12-02T05:17:55,835 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3581974ef1a427f90e086feb86d4f41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733116675206 2024-12-02T05:17:55,837 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/B of 2bec4a5721d5a74dad6f846b3202c90f into a8ce8eef8fb149d08b433d011db86865(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:55,837 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:55,837 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/B, priority=13, startTime=1733116675386; duration=0sec 2024-12-02T05:17:55,837 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:55,837 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:B 2024-12-02T05:17:55,840 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#C#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:55,841 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/2ecca740e11949d59992147bacdf3207 is 50, key is test_row_0/C:col10/1733116675210/Put/seqid=0 2024-12-02T05:17:55,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742091_1267 (size=12663) 2024-12-02T05:17:55,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/0039dedad5f3473fb9488d0de2979c9e 2024-12-02T05:17:55,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/8aa01cfffac64b26a14b4b8167c4a5b8 is 50, key is test_row_0/C:col10/1733116675318/Put/seqid=0 2024-12-02T05:17:55,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742092_1268 (size=12301) 2024-12-02T05:17:55,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116735957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116736067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:56,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116736068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116736078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T05:17:56,198 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:56,198 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-02T05:17:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T05:17:56,200 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:56,200 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:56,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:56,249 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/2ecca740e11949d59992147bacdf3207 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2ecca740e11949d59992147bacdf3207 2024-12-02T05:17:56,253 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/C of 2bec4a5721d5a74dad6f846b3202c90f into 2ecca740e11949d59992147bacdf3207(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:56,254 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:56,254 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/C, priority=13, startTime=1733116675386; duration=0sec 2024-12-02T05:17:56,254 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:56,254 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:C 2024-12-02T05:17:56,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/8aa01cfffac64b26a14b4b8167c4a5b8 2024-12-02T05:17:56,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/4c56febae6384a1ba1158e23892c086e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/4c56febae6384a1ba1158e23892c086e 2024-12-02T05:17:56,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/4c56febae6384a1ba1158e23892c086e, entries=200, sequenceid=264, filesize=14.4 K 2024-12-02T05:17:56,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/0039dedad5f3473fb9488d0de2979c9e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/0039dedad5f3473fb9488d0de2979c9e 2024-12-02T05:17:56,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/0039dedad5f3473fb9488d0de2979c9e, entries=150, sequenceid=264, filesize=12.0 K 2024-12-02T05:17:56,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/8aa01cfffac64b26a14b4b8167c4a5b8 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/8aa01cfffac64b26a14b4b8167c4a5b8 2024-12-02T05:17:56,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/8aa01cfffac64b26a14b4b8167c4a5b8, entries=150, sequenceid=264, filesize=12.0 K 2024-12-02T05:17:56,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 2bec4a5721d5a74dad6f846b3202c90f in 852ms, sequenceid=264, compaction requested=false 2024-12-02T05:17:56,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T05:17:56,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T05:17:56,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:56,353 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-02T05:17:56,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:56,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:56,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:56,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:56,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:56,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/71fc203e433246df9c981b93f451837a is 50, key is test_row_0/A:col10/1733116675435/Put/seqid=0 2024-12-02T05:17:56,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742093_1269 (size=9857) 2024-12-02T05:17:56,371 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/71fc203e433246df9c981b93f451837a 2024-12-02T05:17:56,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/36d39dc3bd074f72b4284578a1db3b7c is 50, key is test_row_0/B:col10/1733116675435/Put/seqid=0 2024-12-02T05:17:56,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742094_1270 (size=9857) 2024-12-02T05:17:56,392 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/36d39dc3bd074f72b4284578a1db3b7c 2024-12-02T05:17:56,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/287e17915fc54f548cd3cda159b1e2c8 is 50, key is test_row_0/C:col10/1733116675435/Put/seqid=0 2024-12-02T05:17:56,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742095_1271 (size=9857) 2024-12-02T05:17:56,419 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/287e17915fc54f548cd3cda159b1e2c8 2024-12-02T05:17:56,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/71fc203e433246df9c981b93f451837a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/71fc203e433246df9c981b93f451837a 2024-12-02T05:17:56,434 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/71fc203e433246df9c981b93f451837a, entries=100, sequenceid=276, filesize=9.6 K 2024-12-02T05:17:56,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/36d39dc3bd074f72b4284578a1db3b7c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/36d39dc3bd074f72b4284578a1db3b7c 2024-12-02T05:17:56,440 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/36d39dc3bd074f72b4284578a1db3b7c, entries=100, sequenceid=276, filesize=9.6 K 2024-12-02T05:17:56,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/287e17915fc54f548cd3cda159b1e2c8 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/287e17915fc54f548cd3cda159b1e2c8 2024-12-02T05:17:56,445 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/287e17915fc54f548cd3cda159b1e2c8, entries=100, sequenceid=276, filesize=9.6 K 2024-12-02T05:17:56,446 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 2bec4a5721d5a74dad6f846b3202c90f in 93ms, sequenceid=276, compaction requested=true 2024-12-02T05:17:56,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:56,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:56,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-02T05:17:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-02T05:17:56,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-02T05:17:56,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 247 msec 2024-12-02T05:17:56,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 252 msec 2024-12-02T05:17:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:56,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T05:17:56,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:56,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:56,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:56,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:56,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:56,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:56,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/0f19ba1b67b040dda06674fac0b0442b is 50, key is test_row_0/A:col10/1733116676482/Put/seqid=0 2024-12-02T05:17:56,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742096_1272 (size=9857) 2024-12-02T05:17:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T05:17:56,517 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:56,518 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-02T05:17:56,519 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T05:17:56,520 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:56,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:56,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116676525, max timestamp=1733116676551Num uniques 859; 2024-12-02T05:17:56,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:17:56,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116676525, max timestamp=1733116676551Num uniques 859; 2024-12-02T05:17:56,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:17:56,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116676525, max timestamp=1733116676551Num uniques 859; 2024-12-02T05:17:56,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:17:56,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116736592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116736595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:56,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:56,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116736596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:56,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116736597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T05:17:56,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T05:17:56,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:56,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:56,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:56,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:56,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:56,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116736707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116736707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116736707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116736707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T05:17:56,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:56,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:56,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:56,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:56,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T05:17:56,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/0f19ba1b67b040dda06674fac0b0442b 2024-12-02T05:17:56,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116736917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116736917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:56,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/12a9249279754d49bd2107749818bd8d is 50, key is test_row_0/B:col10/1733116676482/Put/seqid=0 2024-12-02T05:17:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116736917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:56,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116736920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:56,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742097_1273 (size=9857) 2024-12-02T05:17:56,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T05:17:56,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:56,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:56,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:56,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:56,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:57,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T05:17:57,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:57,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:57,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:57,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:57,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T05:17:57,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116737227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:57,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116737227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:57,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116737227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116737227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:57,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T05:17:57,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:57,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:57,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:57,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:57,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:57,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/12a9249279754d49bd2107749818bd8d 2024-12-02T05:17:57,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/2d4b5c0e8e304f0d986826aadc17a6c6 is 50, key is test_row_0/C:col10/1733116676482/Put/seqid=0 2024-12-02T05:17:57,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742098_1274 (size=9857) 2024-12-02T05:17:57,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/2d4b5c0e8e304f0d986826aadc17a6c6 2024-12-02T05:17:57,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/0f19ba1b67b040dda06674fac0b0442b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0f19ba1b67b040dda06674fac0b0442b 2024-12-02T05:17:57,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0f19ba1b67b040dda06674fac0b0442b, entries=100, sequenceid=286, filesize=9.6 K 2024-12-02T05:17:57,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/12a9249279754d49bd2107749818bd8d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/12a9249279754d49bd2107749818bd8d 2024-12-02T05:17:57,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/12a9249279754d49bd2107749818bd8d, entries=100, sequenceid=286, filesize=9.6 K 2024-12-02T05:17:57,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/2d4b5c0e8e304f0d986826aadc17a6c6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2d4b5c0e8e304f0d986826aadc17a6c6 2024-12-02T05:17:57,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2d4b5c0e8e304f0d986826aadc17a6c6, entries=100, sequenceid=286, filesize=9.6 K 2024-12-02T05:17:57,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 2bec4a5721d5a74dad6f846b3202c90f in 920ms, sequenceid=286, compaction requested=true 2024-12-02T05:17:57,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:57,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:17:57,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:57,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:17:57,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:57,404 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:17:57,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:17:57,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T05:17:57,404 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:17:57,408 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44678 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:17:57,408 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47118 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:17:57,408 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/A is initiating minor compaction (all files) 2024-12-02T05:17:57,408 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/B is initiating minor compaction (all files) 2024-12-02T05:17:57,408 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/B in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:57,408 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/A in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:57,408 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/a8ce8eef8fb149d08b433d011db86865, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/0039dedad5f3473fb9488d0de2979c9e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/36d39dc3bd074f72b4284578a1db3b7c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/12a9249279754d49bd2107749818bd8d] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=43.6 K 2024-12-02T05:17:57,408 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/aa6dfd7e501046d2bface455a06a89a6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/4c56febae6384a1ba1158e23892c086e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/71fc203e433246df9c981b93f451837a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0f19ba1b67b040dda06674fac0b0442b] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=46.0 K 2024-12-02T05:17:57,408 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting a8ce8eef8fb149d08b433d011db86865, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733116675206 2024-12-02T05:17:57,409 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa6dfd7e501046d2bface455a06a89a6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733116675206 2024-12-02T05:17:57,409 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 0039dedad5f3473fb9488d0de2979c9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733116675316 2024-12-02T05:17:57,410 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c56febae6384a1ba1158e23892c086e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733116675316 2024-12-02T05:17:57,410 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 36d39dc3bd074f72b4284578a1db3b7c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733116675435 2024-12-02T05:17:57,410 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71fc203e433246df9c981b93f451837a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733116675435 2024-12-02T05:17:57,410 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 12a9249279754d49bd2107749818bd8d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733116676477 2024-12-02T05:17:57,410 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f19ba1b67b040dda06674fac0b0442b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733116676477 2024-12-02T05:17:57,422 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#A#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:57,422 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/8d409de6c03c472997a2f1cb7e0aee06 is 50, key is test_row_0/A:col10/1733116676482/Put/seqid=0 2024-12-02T05:17:57,426 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#B#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:57,427 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/bfa83170b0ff4bac9bfbf4d0e9dcf37a is 50, key is test_row_0/B:col10/1733116676482/Put/seqid=0 2024-12-02T05:17:57,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T05:17:57,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:57,438 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T05:17:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:17:57,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742099_1275 (size=12949) 2024-12-02T05:17:57,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/6078aedfbf2d45e08b3e49e634859c52 is 50, key is test_row_0/A:col10/1733116676594/Put/seqid=0 2024-12-02T05:17:57,460 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/8d409de6c03c472997a2f1cb7e0aee06 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/8d409de6c03c472997a2f1cb7e0aee06 2024-12-02T05:17:57,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742100_1276 (size=12949) 2024-12-02T05:17:57,470 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/A of 2bec4a5721d5a74dad6f846b3202c90f into 8d409de6c03c472997a2f1cb7e0aee06(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:57,470 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:57,470 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/A, priority=12, startTime=1733116677404; duration=0sec 2024-12-02T05:17:57,470 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:57,470 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:A 2024-12-02T05:17:57,470 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:17:57,471 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44678 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:17:57,472 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/C is initiating minor compaction (all files) 2024-12-02T05:17:57,472 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/C in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:57,472 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2ecca740e11949d59992147bacdf3207, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/8aa01cfffac64b26a14b4b8167c4a5b8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/287e17915fc54f548cd3cda159b1e2c8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2d4b5c0e8e304f0d986826aadc17a6c6] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=43.6 K 2024-12-02T05:17:57,472 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ecca740e11949d59992147bacdf3207, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733116675206 2024-12-02T05:17:57,473 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8aa01cfffac64b26a14b4b8167c4a5b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733116675316 2024-12-02T05:17:57,473 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 287e17915fc54f548cd3cda159b1e2c8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733116675435 2024-12-02T05:17:57,473 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d4b5c0e8e304f0d986826aadc17a6c6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733116676477 2024-12-02T05:17:57,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742101_1277 (size=12301) 2024-12-02T05:17:57,483 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/6078aedfbf2d45e08b3e49e634859c52 2024-12-02T05:17:57,491 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#C#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:57,492 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/3926efffed7a4f8487d3881260b58615 is 50, key is test_row_0/C:col10/1733116676482/Put/seqid=0 2024-12-02T05:17:57,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/7d63f8ebdf9c46ee8832fe1093e97062 is 50, key is test_row_0/B:col10/1733116676594/Put/seqid=0 2024-12-02T05:17:57,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742103_1279 (size=12301) 2024-12-02T05:17:57,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742102_1278 (size=12949) 2024-12-02T05:17:57,529 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/3926efffed7a4f8487d3881260b58615 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3926efffed7a4f8487d3881260b58615 2024-12-02T05:17:57,536 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/C of 2bec4a5721d5a74dad6f846b3202c90f into 3926efffed7a4f8487d3881260b58615(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:57,536 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:57,536 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/C, priority=12, startTime=1733116677404; duration=0sec 2024-12-02T05:17:57,536 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:57,536 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:C 2024-12-02T05:17:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T05:17:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:57,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:57,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:57,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116737745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116737745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:57,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116737745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:57,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116737746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:57,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116737857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116737857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116737857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116737857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:57,866 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/bfa83170b0ff4bac9bfbf4d0e9dcf37a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/bfa83170b0ff4bac9bfbf4d0e9dcf37a 2024-12-02T05:17:57,870 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/B of 2bec4a5721d5a74dad6f846b3202c90f into bfa83170b0ff4bac9bfbf4d0e9dcf37a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:57,870 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:57,870 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/B, priority=12, startTime=1733116677404; duration=0sec 2024-12-02T05:17:57,870 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:57,870 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:B 2024-12-02T05:17:57,917 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/7d63f8ebdf9c46ee8832fe1093e97062 2024-12-02T05:17:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/3364b619e05f4411989ec477e26dd9a4 is 50, key is test_row_0/C:col10/1733116676594/Put/seqid=0 2024-12-02T05:17:57,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742104_1280 (size=12301) 2024-12-02T05:17:58,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116738068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116738068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116738068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116738068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,329 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/3364b619e05f4411989ec477e26dd9a4 2024-12-02T05:17:58,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/6078aedfbf2d45e08b3e49e634859c52 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6078aedfbf2d45e08b3e49e634859c52 2024-12-02T05:17:58,337 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6078aedfbf2d45e08b3e49e634859c52, entries=150, sequenceid=323, filesize=12.0 K 2024-12-02T05:17:58,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/7d63f8ebdf9c46ee8832fe1093e97062 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/7d63f8ebdf9c46ee8832fe1093e97062 2024-12-02T05:17:58,343 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/7d63f8ebdf9c46ee8832fe1093e97062, entries=150, sequenceid=323, filesize=12.0 K 2024-12-02T05:17:58,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/3364b619e05f4411989ec477e26dd9a4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3364b619e05f4411989ec477e26dd9a4 2024-12-02T05:17:58,349 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3364b619e05f4411989ec477e26dd9a4, entries=150, sequenceid=323, filesize=12.0 K 2024-12-02T05:17:58,350 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 2bec4a5721d5a74dad6f846b3202c90f in 912ms, sequenceid=323, compaction requested=false 2024-12-02T05:17:58,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:58,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:58,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-02T05:17:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-02T05:17:58,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-02T05:17:58,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8310 sec 2024-12-02T05:17:58,357 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.8380 sec 2024-12-02T05:17:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:17:58,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T05:17:58,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:58,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:58,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:58,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:58,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:58,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:58,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/0a25e57e4b7c4dd5bc2761f5bf1310fc is 50, key is test_row_0/A:col10/1733116678379/Put/seqid=0 2024-12-02T05:17:58,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116738400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116738403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742105_1281 (size=12301) 2024-12-02T05:17:58,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116738404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116738406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116738517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116738518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116738518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116738518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T05:17:58,667 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:17:58,668 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:17:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-02T05:17:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T05:17:58,669 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:17:58,670 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:17:58,670 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:17:58,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116738727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116738728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116738728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116738728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T05:17:58,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/0a25e57e4b7c4dd5bc2761f5bf1310fc 2024-12-02T05:17:58,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T05:17:58,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/e1d9ff3fd91f46708aa495903c7e8bc9 is 50, key is test_row_0/B:col10/1733116678379/Put/seqid=0 2024-12-02T05:17:58,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:58,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:58,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:58,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:58,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:58,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742106_1282 (size=12301) 2024-12-02T05:17:58,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T05:17:58,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:58,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:58,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:58,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:58,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:58,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T05:17:59,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116739037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:59,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116739038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:59,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116739038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116739038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:59,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T05:17:59,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:59,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:59,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:59,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:59,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:59,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:59,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/e1d9ff3fd91f46708aa495903c7e8bc9 2024-12-02T05:17:59,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/fa2663f204674b3cab1c9d17edeccce8 is 50, key is test_row_0/C:col10/1733116678379/Put/seqid=0 2024-12-02T05:17:59,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742107_1283 (size=12301) 2024-12-02T05:17:59,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T05:17:59,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:59,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:59,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:59,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:59,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T05:17:59,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T05:17:59,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:59,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:59,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:59,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:59,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:59,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116739548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:59,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116739548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:59,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116739548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:17:59,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116739548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:17:59,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:17:59,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:17:59,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T05:17:59,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:59,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:17:59,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:59,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:59,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:17:59,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/fa2663f204674b3cab1c9d17edeccce8 2024-12-02T05:17:59,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/0a25e57e4b7c4dd5bc2761f5bf1310fc as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0a25e57e4b7c4dd5bc2761f5bf1310fc 2024-12-02T05:17:59,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0a25e57e4b7c4dd5bc2761f5bf1310fc, entries=150, sequenceid=339, filesize=12.0 K 2024-12-02T05:17:59,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/e1d9ff3fd91f46708aa495903c7e8bc9 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/e1d9ff3fd91f46708aa495903c7e8bc9 2024-12-02T05:17:59,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/e1d9ff3fd91f46708aa495903c7e8bc9, entries=150, sequenceid=339, filesize=12.0 K 2024-12-02T05:17:59,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/fa2663f204674b3cab1c9d17edeccce8 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/fa2663f204674b3cab1c9d17edeccce8 2024-12-02T05:17:59,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/fa2663f204674b3cab1c9d17edeccce8, entries=150, sequenceid=339, filesize=12.0 K 2024-12-02T05:17:59,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2bec4a5721d5a74dad6f846b3202c90f in 1293ms, sequenceid=339, compaction requested=true 2024-12-02T05:17:59,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:59,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:17:59,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:59,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:17:59,672 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:59,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:59,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:17:59,672 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:59,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:59,673 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:59,673 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:59,673 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/A is initiating minor compaction (all files) 2024-12-02T05:17:59,673 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/B is initiating minor compaction (all files) 2024-12-02T05:17:59,673 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/A in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:59,673 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/B in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:59,674 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/8d409de6c03c472997a2f1cb7e0aee06, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6078aedfbf2d45e08b3e49e634859c52, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0a25e57e4b7c4dd5bc2761f5bf1310fc] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=36.7 K 2024-12-02T05:17:59,674 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/bfa83170b0ff4bac9bfbf4d0e9dcf37a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/7d63f8ebdf9c46ee8832fe1093e97062, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/e1d9ff3fd91f46708aa495903c7e8bc9] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=36.7 K 2024-12-02T05:17:59,674 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d409de6c03c472997a2f1cb7e0aee06, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733116675440 2024-12-02T05:17:59,674 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting bfa83170b0ff4bac9bfbf4d0e9dcf37a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733116675440 2024-12-02T05:17:59,674 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d63f8ebdf9c46ee8832fe1093e97062, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733116676587 2024-12-02T05:17:59,674 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6078aedfbf2d45e08b3e49e634859c52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733116676587 2024-12-02T05:17:59,674 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a25e57e4b7c4dd5bc2761f5bf1310fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733116677744 2024-12-02T05:17:59,674 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting e1d9ff3fd91f46708aa495903c7e8bc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733116677744 2024-12-02T05:17:59,687 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#A#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:59,688 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/44dc78afe19e458ab92efb43f1bbd71f is 50, key is test_row_0/A:col10/1733116678379/Put/seqid=0 2024-12-02T05:17:59,702 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#B#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:59,703 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/9ad0bf9b1a974b1399667b4ef6e9adb3 is 50, key is test_row_0/B:col10/1733116678379/Put/seqid=0 2024-12-02T05:17:59,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742109_1285 (size=13051) 2024-12-02T05:17:59,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742108_1284 (size=13051) 2024-12-02T05:17:59,720 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/9ad0bf9b1a974b1399667b4ef6e9adb3 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/9ad0bf9b1a974b1399667b4ef6e9adb3 2024-12-02T05:17:59,721 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/44dc78afe19e458ab92efb43f1bbd71f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/44dc78afe19e458ab92efb43f1bbd71f 2024-12-02T05:17:59,725 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/A of 2bec4a5721d5a74dad6f846b3202c90f into 44dc78afe19e458ab92efb43f1bbd71f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:59,725 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/B of 2bec4a5721d5a74dad6f846b3202c90f into 9ad0bf9b1a974b1399667b4ef6e9adb3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:17:59,725 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:59,725 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:17:59,725 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/A, priority=13, startTime=1733116679672; duration=0sec 2024-12-02T05:17:59,725 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/B, priority=13, startTime=1733116679672; duration=0sec 2024-12-02T05:17:59,725 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:17:59,725 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:A 2024-12-02T05:17:59,725 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:17:59,725 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:B 2024-12-02T05:17:59,725 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:17:59,726 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:17:59,726 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/C is initiating minor compaction (all files) 2024-12-02T05:17:59,726 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/C in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:59,726 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3926efffed7a4f8487d3881260b58615, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3364b619e05f4411989ec477e26dd9a4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/fa2663f204674b3cab1c9d17edeccce8] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=36.7 K 2024-12-02T05:17:59,726 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3926efffed7a4f8487d3881260b58615, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733116675440 2024-12-02T05:17:59,726 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3364b619e05f4411989ec477e26dd9a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733116676587 2024-12-02T05:17:59,727 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa2663f204674b3cab1c9d17edeccce8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733116677744 2024-12-02T05:17:59,733 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#C#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:17:59,733 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/b3bec0740452489b832223a72e04e771 is 50, key is test_row_0/C:col10/1733116678379/Put/seqid=0 2024-12-02T05:17:59,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T05:17:59,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:17:59,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742110_1286 (size=13051) 2024-12-02T05:17:59,741 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T05:17:59,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:17:59,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:59,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:17:59,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:59,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:17:59,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:17:59,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/07cb9b27048a4f9bab5aeef843fab66a is 50, key is test_row_0/A:col10/1733116678397/Put/seqid=0 2024-12-02T05:17:59,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742111_1287 (size=12301) 2024-12-02T05:17:59,766 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/07cb9b27048a4f9bab5aeef843fab66a 2024-12-02T05:17:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/2ac407fd61ab44e0b6a13211ec33cfb9 is 50, key is test_row_0/B:col10/1733116678397/Put/seqid=0 2024-12-02T05:17:59,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742112_1288 (size=12301) 2024-12-02T05:17:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T05:18:00,146 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/b3bec0740452489b832223a72e04e771 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/b3bec0740452489b832223a72e04e771 2024-12-02T05:18:00,151 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/C of 2bec4a5721d5a74dad6f846b3202c90f into b3bec0740452489b832223a72e04e771(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:00,151 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:00,151 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/C, priority=13, startTime=1733116679672; duration=0sec 2024-12-02T05:18:00,151 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:00,152 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:C 2024-12-02T05:18:00,190 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/2ac407fd61ab44e0b6a13211ec33cfb9 2024-12-02T05:18:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/3864d6e885434021aa55adec7235246c is 50, key is test_row_0/C:col10/1733116678397/Put/seqid=0 2024-12-02T05:18:00,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742113_1289 (size=12301) 2024-12-02T05:18:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:18:00,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:00,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116740570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:00,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116740570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:00,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116740571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:00,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116740571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:00,606 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/3864d6e885434021aa55adec7235246c 2024-12-02T05:18:00,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/07cb9b27048a4f9bab5aeef843fab66a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/07cb9b27048a4f9bab5aeef843fab66a 2024-12-02T05:18:00,614 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/07cb9b27048a4f9bab5aeef843fab66a, entries=150, sequenceid=360, filesize=12.0 K 2024-12-02T05:18:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/2ac407fd61ab44e0b6a13211ec33cfb9 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/2ac407fd61ab44e0b6a13211ec33cfb9 2024-12-02T05:18:00,618 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/2ac407fd61ab44e0b6a13211ec33cfb9, entries=150, sequenceid=360, filesize=12.0 K 2024-12-02T05:18:00,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/3864d6e885434021aa55adec7235246c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3864d6e885434021aa55adec7235246c 2024-12-02T05:18:00,622 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3864d6e885434021aa55adec7235246c, entries=150, sequenceid=360, filesize=12.0 K 2024-12-02T05:18:00,623 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 2bec4a5721d5a74dad6f846b3202c90f in 882ms, sequenceid=360, compaction requested=false 2024-12-02T05:18:00,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:00,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:00,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-02T05:18:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-02T05:18:00,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-02T05:18:00,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9540 sec 2024-12-02T05:18:00,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.9580 sec 2024-12-02T05:18:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:18:00,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T05:18:00,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:18:00,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:00,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:18:00,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:00,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:18:00,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:00,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/c383d502287d4277830d56c2e30b3a2a is 50, key is test_row_0/A:col10/1733116680680/Put/seqid=0 2024-12-02T05:18:00,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742114_1290 (size=14741) 2024-12-02T05:18:00,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/c383d502287d4277830d56c2e30b3a2a 2024-12-02T05:18:00,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/1fbabd9be978403499dc240515fcabfb is 50, key is test_row_0/B:col10/1733116680680/Put/seqid=0 2024-12-02T05:18:00,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116740694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:00,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742115_1291 (size=12301) 2024-12-02T05:18:00,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116740697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:00,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:00,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116740697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:00,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:00,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116740698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:00,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:00,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116740808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:00,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116740808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116740808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116740808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T05:18:00,818 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:00,818 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:18:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-02T05:18:00,820 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:18:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T05:18:00,821 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:18:00,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:18:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T05:18:00,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-02T05:18:00,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:00,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:01,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116741017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116741017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116741018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116741018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/1fbabd9be978403499dc240515fcabfb 2024-12-02T05:18:01,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/5079ea5267de47dea2e777368a43f935 is 50, key is test_row_0/C:col10/1733116680680/Put/seqid=0 2024-12-02T05:18:01,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742116_1292 (size=12301) 2024-12-02T05:18:01,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/5079ea5267de47dea2e777368a43f935 2024-12-02T05:18:01,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/c383d502287d4277830d56c2e30b3a2a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c383d502287d4277830d56c2e30b3a2a 2024-12-02T05:18:01,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c383d502287d4277830d56c2e30b3a2a, entries=200, sequenceid=377, filesize=14.4 K 2024-12-02T05:18:01,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/1fbabd9be978403499dc240515fcabfb as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/1fbabd9be978403499dc240515fcabfb 2024-12-02T05:18:01,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/1fbabd9be978403499dc240515fcabfb, entries=150, sequenceid=377, filesize=12.0 K 2024-12-02T05:18:01,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/5079ea5267de47dea2e777368a43f935 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5079ea5267de47dea2e777368a43f935 2024-12-02T05:18:01,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5079ea5267de47dea2e777368a43f935, entries=150, sequenceid=377, filesize=12.0 K 2024-12-02T05:18:01,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 2bec4a5721d5a74dad6f846b3202c90f in 448ms, sequenceid=377, compaction requested=true 2024-12-02T05:18:01,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:01,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:18:01,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:01,128 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:01,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:18:01,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:01,128 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:01,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:18:01,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:01,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-02T05:18:01,129 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:01,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:01,130 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/B is initiating minor compaction (all files) 2024-12-02T05:18:01,130 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/B in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:01,130 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T05:18:01,130 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/9ad0bf9b1a974b1399667b4ef6e9adb3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/2ac407fd61ab44e0b6a13211ec33cfb9, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/1fbabd9be978403499dc240515fcabfb] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=36.8 K 2024-12-02T05:18:01,130 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:01,130 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/A is initiating minor compaction (all files) 2024-12-02T05:18:01,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:18:01,130 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/A in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:01,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:01,130 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/44dc78afe19e458ab92efb43f1bbd71f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/07cb9b27048a4f9bab5aeef843fab66a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c383d502287d4277830d56c2e30b3a2a] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=39.2 K 2024-12-02T05:18:01,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:18:01,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:01,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:18:01,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:01,131 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44dc78afe19e458ab92efb43f1bbd71f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733116677744 2024-12-02T05:18:01,131 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ad0bf9b1a974b1399667b4ef6e9adb3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733116677744 2024-12-02T05:18:01,131 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07cb9b27048a4f9bab5aeef843fab66a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733116678397 2024-12-02T05:18:01,131 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ac407fd61ab44e0b6a13211ec33cfb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733116678397 2024-12-02T05:18:01,131 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting c383d502287d4277830d56c2e30b3a2a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733116680570 2024-12-02T05:18:01,132 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fbabd9be978403499dc240515fcabfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733116680570 2024-12-02T05:18:01,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/6c4b7d59ad2b4599bc1308436ad2cd4a is 50, key is test_row_0/A:col10/1733116680695/Put/seqid=0 2024-12-02T05:18:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T05:18:01,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742117_1293 (size=9857) 2024-12-02T05:18:01,149 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#A#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:01,150 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/12321e326ec643f08c3bd4be297b618e is 50, key is test_row_0/A:col10/1733116680680/Put/seqid=0 2024-12-02T05:18:01,150 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/6c4b7d59ad2b4599bc1308436ad2cd4a 2024-12-02T05:18:01,152 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:01,153 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/17879430241d4f2185a19dabc2bf67ae is 50, key is test_row_0/B:col10/1733116680680/Put/seqid=0 2024-12-02T05:18:01,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742118_1294 (size=13153) 2024-12-02T05:18:01,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/f9255bf28b414bb09eea70599d487f9e is 50, key is test_row_0/B:col10/1733116680695/Put/seqid=0 2024-12-02T05:18:01,172 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/12321e326ec643f08c3bd4be297b618e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/12321e326ec643f08c3bd4be297b618e 2024-12-02T05:18:01,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742120_1296 (size=9857) 2024-12-02T05:18:01,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742119_1295 (size=13153) 2024-12-02T05:18:01,180 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/A of 2bec4a5721d5a74dad6f846b3202c90f into 12321e326ec643f08c3bd4be297b618e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:01,180 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:01,180 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/A, priority=13, startTime=1733116681128; duration=0sec 2024-12-02T05:18:01,181 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:01,181 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:A 2024-12-02T05:18:01,181 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:01,182 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:01,182 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/C is initiating minor compaction (all files) 2024-12-02T05:18:01,182 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/C in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:01,182 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/b3bec0740452489b832223a72e04e771, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3864d6e885434021aa55adec7235246c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5079ea5267de47dea2e777368a43f935] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=36.8 K 2024-12-02T05:18:01,183 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3bec0740452489b832223a72e04e771, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733116677744 2024-12-02T05:18:01,183 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3864d6e885434021aa55adec7235246c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733116678397 2024-12-02T05:18:01,183 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5079ea5267de47dea2e777368a43f935, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733116680570 2024-12-02T05:18:01,192 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#C#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:01,192 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/2c548e0e9035488282b8443ee1d83bd4 is 50, key is test_row_0/C:col10/1733116680680/Put/seqid=0 2024-12-02T05:18:01,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742121_1297 (size=13153) 2024-12-02T05:18:01,278 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T05:18:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:18:01,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:01,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116741336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116741338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116741339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116741341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T05:18:01,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116741448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116741448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116741448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116741448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,575 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/f9255bf28b414bb09eea70599d487f9e 2024-12-02T05:18:01,579 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/17879430241d4f2185a19dabc2bf67ae as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/17879430241d4f2185a19dabc2bf67ae 2024-12-02T05:18:01,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/a728f3265ac6402da656c39702ae56b3 is 50, key is test_row_0/C:col10/1733116680695/Put/seqid=0 2024-12-02T05:18:01,586 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/B of 2bec4a5721d5a74dad6f846b3202c90f into 17879430241d4f2185a19dabc2bf67ae(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:01,586 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:01,586 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/B, priority=13, startTime=1733116681128; duration=0sec 2024-12-02T05:18:01,586 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:01,586 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:B 2024-12-02T05:18:01,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742122_1298 (size=9857) 2024-12-02T05:18:01,586 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/a728f3265ac6402da656c39702ae56b3 2024-12-02T05:18:01,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/6c4b7d59ad2b4599bc1308436ad2cd4a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6c4b7d59ad2b4599bc1308436ad2cd4a 2024-12-02T05:18:01,595 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6c4b7d59ad2b4599bc1308436ad2cd4a, entries=100, sequenceid=394, filesize=9.6 K 2024-12-02T05:18:01,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/f9255bf28b414bb09eea70599d487f9e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/f9255bf28b414bb09eea70599d487f9e 2024-12-02T05:18:01,599 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/f9255bf28b414bb09eea70599d487f9e, entries=100, sequenceid=394, filesize=9.6 K 2024-12-02T05:18:01,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/a728f3265ac6402da656c39702ae56b3 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/a728f3265ac6402da656c39702ae56b3 2024-12-02T05:18:01,603 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/a728f3265ac6402da656c39702ae56b3, entries=100, sequenceid=394, filesize=9.6 K 2024-12-02T05:18:01,604 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 2bec4a5721d5a74dad6f846b3202c90f in 474ms, sequenceid=394, compaction requested=false 2024-12-02T05:18:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-02T05:18:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-02T05:18:01,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-02T05:18:01,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 785 msec 2024-12-02T05:18:01,608 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/2c548e0e9035488282b8443ee1d83bd4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2c548e0e9035488282b8443ee1d83bd4 2024-12-02T05:18:01,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 791 msec 2024-12-02T05:18:01,613 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/C of 2bec4a5721d5a74dad6f846b3202c90f into 2c548e0e9035488282b8443ee1d83bd4(size=12.8 K), total size for store is 22.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:01,613 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:01,613 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/C, priority=13, startTime=1733116681128; duration=0sec 2024-12-02T05:18:01,613 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:01,613 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:C 2024-12-02T05:18:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:18:01,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T05:18:01,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:18:01,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:01,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:18:01,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:01,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:18:01,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:01,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/9a392a82518a4a7dbc26d2bf7c020c6d is 50, key is test_row_0/A:col10/1733116681659/Put/seqid=0 2024-12-02T05:18:01,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742123_1299 (size=14741) 2024-12-02T05:18:01,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116741672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116741673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116741677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116741678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116741787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116741787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116741788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116741788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T05:18:01,958 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:01,958 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:18:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-02T05:18:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T05:18:01,960 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:18:01,960 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:18:01,961 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:18:01,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116741997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:01,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116741997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:02,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116741997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116741997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T05:18:02,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/9a392a82518a4a7dbc26d2bf7c020c6d 2024-12-02T05:18:02,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/d132c358d0ea4ebe8e0b3b8996028895 is 50, key is test_row_0/B:col10/1733116681659/Put/seqid=0 2024-12-02T05:18:02,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742124_1300 (size=12301) 2024-12-02T05:18:02,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T05:18:02,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:02,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:02,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:02,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T05:18:02,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:02,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:02,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:02,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T05:18:02,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116742308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116742308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:02,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116742308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:02,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116742308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:02,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T05:18:02,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:02,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:02,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:02,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/d132c358d0ea4ebe8e0b3b8996028895 2024-12-02T05:18:02,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/eb1f64285bae442aa3ae203b86c725f7 is 50, key is test_row_0/C:col10/1733116681659/Put/seqid=0 2024-12-02T05:18:02,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742125_1301 (size=12301) 2024-12-02T05:18:02,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T05:18:02,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:02,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:02,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:02,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T05:18:02,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T05:18:02,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:02,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:02,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:02,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116742817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116742817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116742817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:02,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116742817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:02,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T05:18:02,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:02,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:02,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:02,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:02,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/eb1f64285bae442aa3ae203b86c725f7 2024-12-02T05:18:02,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/9a392a82518a4a7dbc26d2bf7c020c6d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/9a392a82518a4a7dbc26d2bf7c020c6d 2024-12-02T05:18:02,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/9a392a82518a4a7dbc26d2bf7c020c6d, entries=200, sequenceid=415, filesize=14.4 K 2024-12-02T05:18:02,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/d132c358d0ea4ebe8e0b3b8996028895 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/d132c358d0ea4ebe8e0b3b8996028895 2024-12-02T05:18:02,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/d132c358d0ea4ebe8e0b3b8996028895, entries=150, sequenceid=415, filesize=12.0 K 2024-12-02T05:18:02,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/eb1f64285bae442aa3ae203b86c725f7 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/eb1f64285bae442aa3ae203b86c725f7 2024-12-02T05:18:02,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/eb1f64285bae442aa3ae203b86c725f7, entries=150, sequenceid=415, filesize=12.0 K 2024-12-02T05:18:02,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 2bec4a5721d5a74dad6f846b3202c90f in 1258ms, sequenceid=415, compaction requested=true 2024-12-02T05:18:02,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:02,920 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:02,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:18:02,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:02,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:18:02,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:02,920 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:02,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:18:02,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:02,921 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37751 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:02,921 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:02,921 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/B is initiating minor compaction (all files) 2024-12-02T05:18:02,921 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/A is initiating minor compaction (all files) 2024-12-02T05:18:02,921 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/B in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:02,921 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/A in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:02,921 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/17879430241d4f2185a19dabc2bf67ae, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/f9255bf28b414bb09eea70599d487f9e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/d132c358d0ea4ebe8e0b3b8996028895] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=34.5 K 2024-12-02T05:18:02,921 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/12321e326ec643f08c3bd4be297b618e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6c4b7d59ad2b4599bc1308436ad2cd4a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/9a392a82518a4a7dbc26d2bf7c020c6d] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=36.9 K 2024-12-02T05:18:02,921 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12321e326ec643f08c3bd4be297b618e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733116680570 2024-12-02T05:18:02,921 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 17879430241d4f2185a19dabc2bf67ae, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733116680570 2024-12-02T05:18:02,921 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c4b7d59ad2b4599bc1308436ad2cd4a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733116680695 2024-12-02T05:18:02,921 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting f9255bf28b414bb09eea70599d487f9e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733116680695 2024-12-02T05:18:02,922 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a392a82518a4a7dbc26d2bf7c020c6d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733116681334 2024-12-02T05:18:02,922 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting d132c358d0ea4ebe8e0b3b8996028895, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733116681334 2024-12-02T05:18:02,927 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#B#compaction#261 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:02,927 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#A#compaction#260 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:02,928 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/aa8fbbb152ab42a6b09d97660f156953 is 50, key is test_row_0/A:col10/1733116681659/Put/seqid=0 2024-12-02T05:18:02,928 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/6a8dfdc9d50e40b1a40cd9fa86aebbdb is 50, key is test_row_0/B:col10/1733116681659/Put/seqid=0 2024-12-02T05:18:02,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742126_1302 (size=13255) 2024-12-02T05:18:02,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742127_1303 (size=13255) 2024-12-02T05:18:02,942 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/6a8dfdc9d50e40b1a40cd9fa86aebbdb as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/6a8dfdc9d50e40b1a40cd9fa86aebbdb 2024-12-02T05:18:02,948 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/B of 2bec4a5721d5a74dad6f846b3202c90f into 6a8dfdc9d50e40b1a40cd9fa86aebbdb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:02,948 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:02,948 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/B, priority=13, startTime=1733116682920; duration=0sec 2024-12-02T05:18:02,948 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:02,948 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:B 2024-12-02T05:18:02,948 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:02,951 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:02,951 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/C is initiating minor compaction (all files) 2024-12-02T05:18:02,951 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/C in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:02,951 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2c548e0e9035488282b8443ee1d83bd4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/a728f3265ac6402da656c39702ae56b3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/eb1f64285bae442aa3ae203b86c725f7] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=34.5 K 2024-12-02T05:18:02,952 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c548e0e9035488282b8443ee1d83bd4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733116680570 2024-12-02T05:18:02,952 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting a728f3265ac6402da656c39702ae56b3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733116680695 2024-12-02T05:18:02,952 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting eb1f64285bae442aa3ae203b86c725f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733116681334 2024-12-02T05:18:02,958 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#C#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:02,958 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/ba62c50045744f3192028dd1228153e2 is 50, key is test_row_0/C:col10/1733116681659/Put/seqid=0 2024-12-02T05:18:02,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742128_1304 (size=13255) 2024-12-02T05:18:03,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T05:18:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:03,028 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T05:18:03,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:18:03,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:03,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:18:03,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:03,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:18:03,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:03,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/6c9db89454694633bf15b231742f009c is 50, key is test_row_0/A:col10/1733116681676/Put/seqid=0 2024-12-02T05:18:03,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742129_1305 (size=12301) 2024-12-02T05:18:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T05:18:03,341 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/aa8fbbb152ab42a6b09d97660f156953 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/aa8fbbb152ab42a6b09d97660f156953 2024-12-02T05:18:03,345 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/A of 2bec4a5721d5a74dad6f846b3202c90f into aa8fbbb152ab42a6b09d97660f156953(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:03,345 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:03,345 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/A, priority=13, startTime=1733116682919; duration=0sec 2024-12-02T05:18:03,345 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:03,346 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:A 2024-12-02T05:18:03,367 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/ba62c50045744f3192028dd1228153e2 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/ba62c50045744f3192028dd1228153e2 2024-12-02T05:18:03,371 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/C of 2bec4a5721d5a74dad6f846b3202c90f into ba62c50045744f3192028dd1228153e2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:03,371 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:03,371 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/C, priority=13, startTime=1733116682920; duration=0sec 2024-12-02T05:18:03,371 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:03,371 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:C 2024-12-02T05:18:03,442 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/6c9db89454694633bf15b231742f009c 2024-12-02T05:18:03,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/18d8a88f1274443a81b231aa277d249c is 50, key is test_row_0/B:col10/1733116681676/Put/seqid=0 2024-12-02T05:18:03,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742130_1306 (size=12301) 2024-12-02T05:18:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:18:03,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:03,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116743839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:03,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116743840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:03,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:03,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116743843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116743843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:03,861 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/18d8a88f1274443a81b231aa277d249c 2024-12-02T05:18:03,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/bc3c49ce835047dabedda2eb8c2a433c is 50, key is test_row_0/C:col10/1733116681676/Put/seqid=0 2024-12-02T05:18:03,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742131_1307 (size=12301) 2024-12-02T05:18:03,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116743948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116743948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:03,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116743957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:03,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116743957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T05:18:04,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116744157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116744157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116744167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116744167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,270 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/bc3c49ce835047dabedda2eb8c2a433c 2024-12-02T05:18:04,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/6c9db89454694633bf15b231742f009c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6c9db89454694633bf15b231742f009c 2024-12-02T05:18:04,278 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6c9db89454694633bf15b231742f009c, entries=150, sequenceid=431, filesize=12.0 K 2024-12-02T05:18:04,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/18d8a88f1274443a81b231aa277d249c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/18d8a88f1274443a81b231aa277d249c 2024-12-02T05:18:04,282 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/18d8a88f1274443a81b231aa277d249c, entries=150, sequenceid=431, filesize=12.0 K 2024-12-02T05:18:04,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/bc3c49ce835047dabedda2eb8c2a433c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/bc3c49ce835047dabedda2eb8c2a433c 2024-12-02T05:18:04,286 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/bc3c49ce835047dabedda2eb8c2a433c, entries=150, sequenceid=431, filesize=12.0 K 2024-12-02T05:18:04,287 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2bec4a5721d5a74dad6f846b3202c90f in 1259ms, sequenceid=431, compaction requested=false 2024-12-02T05:18:04,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:04,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:04,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-02T05:18:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-02T05:18:04,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-02T05:18:04,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3270 sec 2024-12-02T05:18:04,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.3320 sec 2024-12-02T05:18:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:18:04,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T05:18:04,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:18:04,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:04,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:18:04,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:04,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:18:04,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:04,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/192321135d5142bbb8d4a2313456158f is 50, key is test_row_0/A:col10/1733116684468/Put/seqid=0 2024-12-02T05:18:04,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742132_1308 (size=12301) 2024-12-02T05:18:04,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116744482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116744483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116744484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116744486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116744597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116744597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116744598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116744597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49568 deadline: 1733116744787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,791 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in 2bec4a5721d5a74dad6f846b3202c90f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18274 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:18:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116744807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116744807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116744808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116744808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:04,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/192321135d5142bbb8d4a2313456158f 2024-12-02T05:18:04,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/23fcb59339a449168546a19cae7307f6 is 50, key is test_row_0/B:col10/1733116684468/Put/seqid=0 2024-12-02T05:18:04,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742133_1309 (size=12301) 2024-12-02T05:18:04,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/23fcb59339a449168546a19cae7307f6 2024-12-02T05:18:04,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/0584b026f3d14f869a57da678bd36744 is 50, key is test_row_0/C:col10/1733116684468/Put/seqid=0 2024-12-02T05:18:04,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742134_1310 (size=12301) 2024-12-02T05:18:05,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116745117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:05,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116745117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:05,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:05,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116745117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116745117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:05,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/0584b026f3d14f869a57da678bd36744 2024-12-02T05:18:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/192321135d5142bbb8d4a2313456158f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/192321135d5142bbb8d4a2313456158f 2024-12-02T05:18:05,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/192321135d5142bbb8d4a2313456158f, entries=150, sequenceid=451, filesize=12.0 K 2024-12-02T05:18:05,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/23fcb59339a449168546a19cae7307f6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/23fcb59339a449168546a19cae7307f6 2024-12-02T05:18:05,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/23fcb59339a449168546a19cae7307f6, entries=150, sequenceid=451, filesize=12.0 K 2024-12-02T05:18:05,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/0584b026f3d14f869a57da678bd36744 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/0584b026f3d14f869a57da678bd36744 2024-12-02T05:18:05,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/0584b026f3d14f869a57da678bd36744, entries=150, sequenceid=451, filesize=12.0 K 2024-12-02T05:18:05,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2bec4a5721d5a74dad6f846b3202c90f in 842ms, sequenceid=451, compaction requested=true 2024-12-02T05:18:05,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:05,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:18:05,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:05,311 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:05,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:18:05,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:05,311 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:05,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2bec4a5721d5a74dad6f846b3202c90f:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:18:05,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:05,312 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:05,312 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:05,313 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/B is initiating minor compaction (all files) 2024-12-02T05:18:05,313 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/A is initiating minor compaction (all files) 2024-12-02T05:18:05,313 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/B in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:05,313 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/A in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:05,313 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/aa8fbbb152ab42a6b09d97660f156953, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6c9db89454694633bf15b231742f009c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/192321135d5142bbb8d4a2313456158f] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=37.0 K 2024-12-02T05:18:05,313 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/6a8dfdc9d50e40b1a40cd9fa86aebbdb, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/18d8a88f1274443a81b231aa277d249c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/23fcb59339a449168546a19cae7307f6] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=37.0 K 2024-12-02T05:18:05,313 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa8fbbb152ab42a6b09d97660f156953, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733116681334 2024-12-02T05:18:05,313 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a8dfdc9d50e40b1a40cd9fa86aebbdb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733116681334 2024-12-02T05:18:05,313 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c9db89454694633bf15b231742f009c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733116681672 2024-12-02T05:18:05,313 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 18d8a88f1274443a81b231aa277d249c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733116681672 2024-12-02T05:18:05,314 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 23fcb59339a449168546a19cae7307f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733116683838 2024-12-02T05:18:05,314 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 192321135d5142bbb8d4a2313456158f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733116683838 2024-12-02T05:18:05,322 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#B#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:05,323 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/d5031e7f49164f10a4f383963ba9fdce is 50, key is test_row_0/B:col10/1733116684468/Put/seqid=0 2024-12-02T05:18:05,325 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#A#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:05,325 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/f17a6f302e0a41a98b26a71ea7acc85c is 50, key is test_row_0/A:col10/1733116684468/Put/seqid=0 2024-12-02T05:18:05,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742135_1311 (size=13357) 2024-12-02T05:18:05,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742136_1312 (size=13357) 2024-12-02T05:18:05,341 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/f17a6f302e0a41a98b26a71ea7acc85c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/f17a6f302e0a41a98b26a71ea7acc85c 2024-12-02T05:18:05,345 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/A of 2bec4a5721d5a74dad6f846b3202c90f into f17a6f302e0a41a98b26a71ea7acc85c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:05,345 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:05,345 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/A, priority=13, startTime=1733116685311; duration=0sec 2024-12-02T05:18:05,345 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:05,346 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:A 2024-12-02T05:18:05,346 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:05,347 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:05,347 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 2bec4a5721d5a74dad6f846b3202c90f/C is initiating minor compaction (all files) 2024-12-02T05:18:05,347 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2bec4a5721d5a74dad6f846b3202c90f/C in TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:05,347 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/ba62c50045744f3192028dd1228153e2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/bc3c49ce835047dabedda2eb8c2a433c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/0584b026f3d14f869a57da678bd36744] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp, totalSize=37.0 K 2024-12-02T05:18:05,347 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba62c50045744f3192028dd1228153e2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733116681334 2024-12-02T05:18:05,347 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc3c49ce835047dabedda2eb8c2a433c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733116681672 2024-12-02T05:18:05,348 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0584b026f3d14f869a57da678bd36744, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733116683838 2024-12-02T05:18:05,359 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2bec4a5721d5a74dad6f846b3202c90f#C#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:05,359 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/63f847f23c7a4e20b7fe7f990b2543ba is 50, key is test_row_0/C:col10/1733116684468/Put/seqid=0 2024-12-02T05:18:05,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742137_1313 (size=13357) 2024-12-02T05:18:05,777 INFO [AsyncFSWAL-0-hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c-prefix:f49461030c22,46495,1733116594459 {}] wal.AbstractFSWAL(1368): Slow sync cost: 147 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34519,DS-5a5c0e22-a56e-4f32-ae1e-a90082acf4bf,DISK]] 2024-12-02T05:18:05,777 INFO [AsyncFSWAL-0-hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c-prefix:f49461030c22,46495,1733116594459 {}] wal.AbstractFSWAL(1368): Slow sync cost: 148 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34519,DS-5a5c0e22-a56e-4f32-ae1e-a90082acf4bf,DISK]] 2024-12-02T05:18:05,778 INFO [AsyncFSWAL-0-hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c-prefix:f49461030c22,46495,1733116594459 {}] wal.AbstractFSWAL(1368): Slow sync cost: 147 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34519,DS-5a5c0e22-a56e-4f32-ae1e-a90082acf4bf,DISK]] 2024-12-02T05:18:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:18:05,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T05:18:05,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:18:05,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:05,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:18:05,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:05,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:18:05,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:05,784 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/63f847f23c7a4e20b7fe7f990b2543ba as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/63f847f23c7a4e20b7fe7f990b2543ba 2024-12-02T05:18:05,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/982174d4d15d41a99bc7671501b05189 is 50, key is test_row_0/A:col10/1733116684484/Put/seqid=0 2024-12-02T05:18:05,789 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/d5031e7f49164f10a4f383963ba9fdce as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/d5031e7f49164f10a4f383963ba9fdce 2024-12-02T05:18:05,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742138_1314 (size=17181) 2024-12-02T05:18:05,797 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/B of 2bec4a5721d5a74dad6f846b3202c90f into d5031e7f49164f10a4f383963ba9fdce(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:05,797 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2bec4a5721d5a74dad6f846b3202c90f/C of 2bec4a5721d5a74dad6f846b3202c90f into 63f847f23c7a4e20b7fe7f990b2543ba(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:05,797 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:05,797 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:05,797 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/B, priority=13, startTime=1733116685311; duration=0sec 2024-12-02T05:18:05,797 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., storeName=2bec4a5721d5a74dad6f846b3202c90f/C, priority=13, startTime=1733116685311; duration=0sec 2024-12-02T05:18:05,797 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:05,797 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:C 2024-12-02T05:18:05,797 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:05,797 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2bec4a5721d5a74dad6f846b3202c90f:B 2024-12-02T05:18:05,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:05,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116745803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116745803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:05,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116745804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116745804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:05,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116745917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:05,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116745917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116745918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116745918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T05:18:06,117 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:06,118 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:18:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-12-02T05:18:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T05:18:06,119 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:18:06,120 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:18:06,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:18:06,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:06,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116746127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116746127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:06,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116746127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:06,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116746128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:06,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/982174d4d15d41a99bc7671501b05189 2024-12-02T05:18:06,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/5b1cb019c8304e83bbdd05b2cda5afc1 is 50, key is test_row_0/B:col10/1733116684484/Put/seqid=0 2024-12-02T05:18:06,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742139_1315 (size=12301) 2024-12-02T05:18:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T05:18:06,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T05:18:06,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:06,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:06,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:06,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:06,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:06,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T05:18:06,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:06,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:06,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:06,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:06,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T05:18:06,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116746437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:06,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116746437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116746437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116746437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:06,454 INFO [Thread-1040 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1040. 2024-12-02T05:18:06,454 DEBUG [Thread-1040 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:06,454 DEBUG [Thread-1040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:06,454 DEBUG [Thread-1040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:06,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:06,455 INFO [Thread-1038 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1038. 2024-12-02T05:18:06,455 DEBUG [Thread-1038 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:06,455 DEBUG [Thread-1038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:06,455 DEBUG [Thread-1038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:06,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:06,456 INFO [Thread-1042 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1042. 2024-12-02T05:18:06,456 DEBUG [Thread-1042 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:06,456 DEBUG [Thread-1042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:06,456 INFO [Thread-1044 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1044. 2024-12-02T05:18:06,456 DEBUG [Thread-1044 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:06,456 DEBUG [Thread-1044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:06,456 DEBUG [Thread-1042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:06,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:06,456 DEBUG [Thread-1044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:06,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:06,458 INFO [Thread-1046 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1046. 2024-12-02T05:18:06,458 DEBUG [Thread-1046 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:06,458 DEBUG [Thread-1046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:06,458 DEBUG [Thread-1046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:06,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:06,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T05:18:06,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:06,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:06,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:06,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:06,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:06,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/5b1cb019c8304e83bbdd05b2cda5afc1 2024-12-02T05:18:06,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/13a37c9e87584c63aaf8fc50898765b9 is 50, key is test_row_0/C:col10/1733116684484/Put/seqid=0 2024-12-02T05:18:06,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742140_1316 (size=12301) 2024-12-02T05:18:06,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T05:18:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:06,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T05:18:06,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T05:18:06,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:06,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:06,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:06,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:06,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:06,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:06,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:06,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1733116746948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49618 deadline: 1733116746948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:06,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49598 deadline: 1733116746949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:06,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:06,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1733116746949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:07,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/13a37c9e87584c63aaf8fc50898765b9 2024-12-02T05:18:07,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/982174d4d15d41a99bc7671501b05189 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/982174d4d15d41a99bc7671501b05189 2024-12-02T05:18:07,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/982174d4d15d41a99bc7671501b05189, entries=250, sequenceid=470, filesize=16.8 K 2024-12-02T05:18:07,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/5b1cb019c8304e83bbdd05b2cda5afc1 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/5b1cb019c8304e83bbdd05b2cda5afc1 2024-12-02T05:18:07,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/5b1cb019c8304e83bbdd05b2cda5afc1, entries=150, sequenceid=470, filesize=12.0 K 2024-12-02T05:18:07,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/13a37c9e87584c63aaf8fc50898765b9 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/13a37c9e87584c63aaf8fc50898765b9 2024-12-02T05:18:07,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/13a37c9e87584c63aaf8fc50898765b9, entries=150, sequenceid=470, filesize=12.0 K 2024-12-02T05:18:07,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2bec4a5721d5a74dad6f846b3202c90f in 1254ms, sequenceid=470, compaction requested=false 2024-12-02T05:18:07,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:07,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T05:18:07,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:07,040 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T05:18:07,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:18:07,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:07,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:18:07,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:07,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:18:07,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:07,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/0a03c3be12dc479cb194e233a92dd8e7 is 50, key is test_row_0/A:col10/1733116685803/Put/seqid=0 2024-12-02T05:18:07,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742141_1317 (size=12301) 2024-12-02T05:18:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T05:18:07,448 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/0a03c3be12dc479cb194e233a92dd8e7 2024-12-02T05:18:07,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/c5d5392aad8841f784fb73764c9be895 is 50, key is test_row_0/B:col10/1733116685803/Put/seqid=0 2024-12-02T05:18:07,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742142_1318 (size=12301) 2024-12-02T05:18:07,861 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/c5d5392aad8841f784fb73764c9be895 2024-12-02T05:18:07,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/7c8b95b33e9f4e72b70e5d0e7bda3491 is 50, key is test_row_0/C:col10/1733116685803/Put/seqid=0 2024-12-02T05:18:07,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742143_1319 (size=12301) 2024-12-02T05:18:07,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:18:07,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. as already flushing 2024-12-02T05:18:07,961 INFO [Thread-1031 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1031. 2024-12-02T05:18:07,961 INFO [Thread-1033 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1033. 2024-12-02T05:18:07,961 DEBUG [Thread-1031 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:07,961 DEBUG [Thread-1033 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:07,962 DEBUG [Thread-1031 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:07,962 DEBUG [Thread-1033 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:07,962 INFO [Thread-1027 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1027. 2024-12-02T05:18:07,962 DEBUG [Thread-1027 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:07,962 DEBUG [Thread-1027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:07,962 DEBUG [Thread-1031 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:07,962 DEBUG [Thread-1033 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:07,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:07,962 DEBUG [Thread-1027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:07,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:07,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:07,969 INFO [Thread-1035 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1035. 2024-12-02T05:18:07,969 DEBUG [Thread-1035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:07,969 DEBUG [Thread-1035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:07,969 DEBUG [Thread-1035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:07,969 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T05:18:08,275 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/7c8b95b33e9f4e72b70e5d0e7bda3491 2024-12-02T05:18:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/0a03c3be12dc479cb194e233a92dd8e7 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0a03c3be12dc479cb194e233a92dd8e7 2024-12-02T05:18:08,286 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0a03c3be12dc479cb194e233a92dd8e7, entries=150, sequenceid=486, filesize=12.0 K 2024-12-02T05:18:08,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/c5d5392aad8841f784fb73764c9be895 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/c5d5392aad8841f784fb73764c9be895 2024-12-02T05:18:08,291 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/c5d5392aad8841f784fb73764c9be895, entries=150, sequenceid=486, filesize=12.0 K 2024-12-02T05:18:08,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/7c8b95b33e9f4e72b70e5d0e7bda3491 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/7c8b95b33e9f4e72b70e5d0e7bda3491 2024-12-02T05:18:08,297 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/7c8b95b33e9f4e72b70e5d0e7bda3491, entries=150, sequenceid=486, filesize=12.0 K 2024-12-02T05:18:08,298 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=26.84 KB/27480 for 2bec4a5721d5a74dad6f846b3202c90f in 1258ms, sequenceid=486, compaction requested=true 2024-12-02T05:18:08,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 2bec4a5721d5a74dad6f846b3202c90f: 2024-12-02T05:18:08,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:08,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-12-02T05:18:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-12-02T05:18:08,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-02T05:18:08,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1790 sec 2024-12-02T05:18:08,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.1830 sec 2024-12-02T05:18:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T05:18:10,279 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:14,841 INFO [Thread-1029 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1029. 2024-12-02T05:18:14,841 DEBUG [Thread-1029 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:14,842 DEBUG [Thread-1029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:14,842 DEBUG [Thread-1029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:14,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T05:18:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-12-02T05:18:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-12-02T05:18:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-12-02T05:18:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-12-02T05:18:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-12-02T05:18:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T05:18:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4169 2024-12-02T05:18:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4259 2024-12-02T05:18:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4323 2024-12-02T05:18:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4159 2024-12-02T05:18:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4242 2024-12-02T05:18:14,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T05:18:14,843 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T05:18:14,843 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T05:18:14,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:14,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:14,843 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:14,844 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T05:18:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T05:18:14,848 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116694848"}]},"ts":"1733116694848"} 2024-12-02T05:18:14,850 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T05:18:14,850 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T05:18:14,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T05:18:14,853 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bec4a5721d5a74dad6f846b3202c90f, UNASSIGN}] 2024-12-02T05:18:14,854 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bec4a5721d5a74dad6f846b3202c90f, UNASSIGN 2024-12-02T05:18:14,855 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=2bec4a5721d5a74dad6f846b3202c90f, regionState=CLOSING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:18:14,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bec4a5721d5a74dad6f846b3202c90f, UNASSIGN because future has completed 2024-12-02T05:18:14,857 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T05:18:14,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:18:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T05:18:15,010 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:18:15,010 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T05:18:15,010 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 2bec4a5721d5a74dad6f846b3202c90f, disabling compactions & flushes 2024-12-02T05:18:15,010 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:15,010 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:15,010 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. after waiting 0 ms 2024-12-02T05:18:15,010 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:15,010 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing 2bec4a5721d5a74dad6f846b3202c90f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T05:18:15,010 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=A 2024-12-02T05:18:15,010 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:15,010 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=B 2024-12-02T05:18:15,010 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:15,010 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2bec4a5721d5a74dad6f846b3202c90f, store=C 2024-12-02T05:18:15,010 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:15,034 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/e7d1ba325d23496daeeb09a5c6707dee is 50, key is test_row_0/A:col10/1733116687968/Put/seqid=0 2024-12-02T05:18:15,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742144_1320 (size=12301) 2024-12-02T05:18:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T05:18:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T05:18:15,543 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/e7d1ba325d23496daeeb09a5c6707dee 2024-12-02T05:18:15,816 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/4287929ad00b40398ad383fa09f75fc4 is 50, key is test_row_0/B:col10/1733116687968/Put/seqid=0 2024-12-02T05:18:15,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742145_1321 (size=12301) 2024-12-02T05:18:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T05:18:16,222 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/4287929ad00b40398ad383fa09f75fc4 2024-12-02T05:18:16,233 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/1099a2a3432b45c6949d72e1e7db30c6 is 50, key is test_row_0/C:col10/1733116687968/Put/seqid=0 2024-12-02T05:18:16,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742146_1322 (size=12301) 2024-12-02T05:18:16,239 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/1099a2a3432b45c6949d72e1e7db30c6 2024-12-02T05:18:16,245 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/A/e7d1ba325d23496daeeb09a5c6707dee as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/e7d1ba325d23496daeeb09a5c6707dee 2024-12-02T05:18:16,250 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/e7d1ba325d23496daeeb09a5c6707dee, entries=150, sequenceid=494, filesize=12.0 K 2024-12-02T05:18:16,251 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/B/4287929ad00b40398ad383fa09f75fc4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/4287929ad00b40398ad383fa09f75fc4 2024-12-02T05:18:16,254 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/4287929ad00b40398ad383fa09f75fc4, entries=150, sequenceid=494, filesize=12.0 K 2024-12-02T05:18:16,255 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/.tmp/C/1099a2a3432b45c6949d72e1e7db30c6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/1099a2a3432b45c6949d72e1e7db30c6 2024-12-02T05:18:16,258 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/1099a2a3432b45c6949d72e1e7db30c6, entries=150, sequenceid=494, filesize=12.0 K 2024-12-02T05:18:16,258 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2bec4a5721d5a74dad6f846b3202c90f in 1248ms, sequenceid=494, compaction requested=true 2024-12-02T05:18:16,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6a65ffe569684ec48a15453f39d9bdfc, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/57ac824115734051b601a32e449ac5a4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/e8be36c39a9842b792689401df308ac6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/5334b8314c6042eaafc2d7df9c9a5582, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c6c32870794446a99f593b1c998fc0b4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/7ed3970cb2424a3582b124cbabe4ca64, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c744f96702af428e81df8bd0f3eb65fa, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/271dbbf536724f1fa3dc4be16d5eba1b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/749850001c464067b51264445456d464, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/2abd8aefbb0b4797af2d507ec03b5a7d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/97d5ec58a75b45ba999d52e5a77790a3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/55649d932d3f44c2852953ab121e4315, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6587565a6b584ec3befdd2d36b2b8987, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/8861f85a53e945738817850eb023eaa1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/798c3a63ab5a46f99601cc4ae18a4357, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/aa6dfd7e501046d2bface455a06a89a6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/4c56febae6384a1ba1158e23892c086e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/71fc203e433246df9c981b93f451837a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/8d409de6c03c472997a2f1cb7e0aee06, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0f19ba1b67b040dda06674fac0b0442b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6078aedfbf2d45e08b3e49e634859c52, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/44dc78afe19e458ab92efb43f1bbd71f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0a25e57e4b7c4dd5bc2761f5bf1310fc, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/07cb9b27048a4f9bab5aeef843fab66a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c383d502287d4277830d56c2e30b3a2a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/12321e326ec643f08c3bd4be297b618e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6c4b7d59ad2b4599bc1308436ad2cd4a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/9a392a82518a4a7dbc26d2bf7c020c6d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/aa8fbbb152ab42a6b09d97660f156953, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6c9db89454694633bf15b231742f009c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/192321135d5142bbb8d4a2313456158f] to archive 2024-12-02T05:18:16,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:18:16,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6a65ffe569684ec48a15453f39d9bdfc to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6a65ffe569684ec48a15453f39d9bdfc 2024-12-02T05:18:16,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/57ac824115734051b601a32e449ac5a4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/57ac824115734051b601a32e449ac5a4 2024-12-02T05:18:16,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/e8be36c39a9842b792689401df308ac6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/e8be36c39a9842b792689401df308ac6 2024-12-02T05:18:16,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/5334b8314c6042eaafc2d7df9c9a5582 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/5334b8314c6042eaafc2d7df9c9a5582 2024-12-02T05:18:16,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c6c32870794446a99f593b1c998fc0b4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c6c32870794446a99f593b1c998fc0b4 2024-12-02T05:18:16,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/7ed3970cb2424a3582b124cbabe4ca64 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/7ed3970cb2424a3582b124cbabe4ca64 2024-12-02T05:18:16,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c744f96702af428e81df8bd0f3eb65fa to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c744f96702af428e81df8bd0f3eb65fa 2024-12-02T05:18:16,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/271dbbf536724f1fa3dc4be16d5eba1b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/271dbbf536724f1fa3dc4be16d5eba1b 2024-12-02T05:18:16,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/749850001c464067b51264445456d464 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/749850001c464067b51264445456d464 2024-12-02T05:18:16,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/2abd8aefbb0b4797af2d507ec03b5a7d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/2abd8aefbb0b4797af2d507ec03b5a7d 2024-12-02T05:18:16,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/97d5ec58a75b45ba999d52e5a77790a3 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/97d5ec58a75b45ba999d52e5a77790a3 2024-12-02T05:18:16,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/55649d932d3f44c2852953ab121e4315 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/55649d932d3f44c2852953ab121e4315 2024-12-02T05:18:16,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6587565a6b584ec3befdd2d36b2b8987 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6587565a6b584ec3befdd2d36b2b8987 2024-12-02T05:18:16,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/8861f85a53e945738817850eb023eaa1 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/8861f85a53e945738817850eb023eaa1 2024-12-02T05:18:16,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/798c3a63ab5a46f99601cc4ae18a4357 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/798c3a63ab5a46f99601cc4ae18a4357 2024-12-02T05:18:16,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/aa6dfd7e501046d2bface455a06a89a6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/aa6dfd7e501046d2bface455a06a89a6 2024-12-02T05:18:16,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/4c56febae6384a1ba1158e23892c086e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/4c56febae6384a1ba1158e23892c086e 2024-12-02T05:18:16,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/71fc203e433246df9c981b93f451837a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/71fc203e433246df9c981b93f451837a 2024-12-02T05:18:16,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/8d409de6c03c472997a2f1cb7e0aee06 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/8d409de6c03c472997a2f1cb7e0aee06 2024-12-02T05:18:16,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0f19ba1b67b040dda06674fac0b0442b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0f19ba1b67b040dda06674fac0b0442b 2024-12-02T05:18:16,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6078aedfbf2d45e08b3e49e634859c52 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6078aedfbf2d45e08b3e49e634859c52 2024-12-02T05:18:16,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/44dc78afe19e458ab92efb43f1bbd71f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/44dc78afe19e458ab92efb43f1bbd71f 2024-12-02T05:18:16,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0a25e57e4b7c4dd5bc2761f5bf1310fc to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0a25e57e4b7c4dd5bc2761f5bf1310fc 2024-12-02T05:18:16,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/07cb9b27048a4f9bab5aeef843fab66a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/07cb9b27048a4f9bab5aeef843fab66a 2024-12-02T05:18:16,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c383d502287d4277830d56c2e30b3a2a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/c383d502287d4277830d56c2e30b3a2a 2024-12-02T05:18:16,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/12321e326ec643f08c3bd4be297b618e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/12321e326ec643f08c3bd4be297b618e 2024-12-02T05:18:16,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6c4b7d59ad2b4599bc1308436ad2cd4a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6c4b7d59ad2b4599bc1308436ad2cd4a 2024-12-02T05:18:16,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/9a392a82518a4a7dbc26d2bf7c020c6d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/9a392a82518a4a7dbc26d2bf7c020c6d 2024-12-02T05:18:16,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/aa8fbbb152ab42a6b09d97660f156953 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/aa8fbbb152ab42a6b09d97660f156953 2024-12-02T05:18:16,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6c9db89454694633bf15b231742f009c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/6c9db89454694633bf15b231742f009c 2024-12-02T05:18:16,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/192321135d5142bbb8d4a2313456158f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/192321135d5142bbb8d4a2313456158f 2024-12-02T05:18:16,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/26d3062617f74cc89f24d0dbcd4f163d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/62a106b4afc641f78bade25c619d1202, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/b24942fbfe4b467a9bfa077d2c5c46ad, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/1747502db267467d95056417b669a5e5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/23d1bf3578bf4076991d0410e86eabe3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/3ad5dc984cb94c11ae62471864c21a64, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/c9180f7c9af04f1f90b6061eac7da8b7, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/628f0cdd383041f091f7ca8c3865f6f7, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/652ac67b2b3e41b0979a72862c55c493, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/870e8e5d496647af912c917415f3c29c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/d5bdc84e30034c6b9ff0224cb0fab444, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/b8066ac2975c42f7aa9aff53da440d84, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/e5c35caa873c452caabcd1d99c11db68, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/0ed20baf1872444a8de5061b3592b9f4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/a8ce8eef8fb149d08b433d011db86865, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/8989b1e9e5594fe886bb3e72c557741a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/0039dedad5f3473fb9488d0de2979c9e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/36d39dc3bd074f72b4284578a1db3b7c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/bfa83170b0ff4bac9bfbf4d0e9dcf37a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/12a9249279754d49bd2107749818bd8d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/7d63f8ebdf9c46ee8832fe1093e97062, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/9ad0bf9b1a974b1399667b4ef6e9adb3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/e1d9ff3fd91f46708aa495903c7e8bc9, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/2ac407fd61ab44e0b6a13211ec33cfb9, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/17879430241d4f2185a19dabc2bf67ae, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/1fbabd9be978403499dc240515fcabfb, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/f9255bf28b414bb09eea70599d487f9e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/6a8dfdc9d50e40b1a40cd9fa86aebbdb, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/d132c358d0ea4ebe8e0b3b8996028895, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/18d8a88f1274443a81b231aa277d249c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/23fcb59339a449168546a19cae7307f6] to archive 2024-12-02T05:18:16,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:18:16,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/26d3062617f74cc89f24d0dbcd4f163d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/26d3062617f74cc89f24d0dbcd4f163d 2024-12-02T05:18:16,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/62a106b4afc641f78bade25c619d1202 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/62a106b4afc641f78bade25c619d1202 2024-12-02T05:18:16,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/b24942fbfe4b467a9bfa077d2c5c46ad to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/b24942fbfe4b467a9bfa077d2c5c46ad 2024-12-02T05:18:16,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/1747502db267467d95056417b669a5e5 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/1747502db267467d95056417b669a5e5 2024-12-02T05:18:16,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/23d1bf3578bf4076991d0410e86eabe3 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/23d1bf3578bf4076991d0410e86eabe3 2024-12-02T05:18:16,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/3ad5dc984cb94c11ae62471864c21a64 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/3ad5dc984cb94c11ae62471864c21a64 2024-12-02T05:18:16,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/c9180f7c9af04f1f90b6061eac7da8b7 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/c9180f7c9af04f1f90b6061eac7da8b7 2024-12-02T05:18:16,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/628f0cdd383041f091f7ca8c3865f6f7 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/628f0cdd383041f091f7ca8c3865f6f7 2024-12-02T05:18:16,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/652ac67b2b3e41b0979a72862c55c493 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/652ac67b2b3e41b0979a72862c55c493 2024-12-02T05:18:16,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/870e8e5d496647af912c917415f3c29c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/870e8e5d496647af912c917415f3c29c 2024-12-02T05:18:16,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/d5bdc84e30034c6b9ff0224cb0fab444 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/d5bdc84e30034c6b9ff0224cb0fab444 2024-12-02T05:18:16,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/b8066ac2975c42f7aa9aff53da440d84 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/b8066ac2975c42f7aa9aff53da440d84 2024-12-02T05:18:16,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/e5c35caa873c452caabcd1d99c11db68 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/e5c35caa873c452caabcd1d99c11db68 2024-12-02T05:18:16,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/0ed20baf1872444a8de5061b3592b9f4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/0ed20baf1872444a8de5061b3592b9f4 2024-12-02T05:18:16,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/a8ce8eef8fb149d08b433d011db86865 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/a8ce8eef8fb149d08b433d011db86865 2024-12-02T05:18:16,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/8989b1e9e5594fe886bb3e72c557741a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/8989b1e9e5594fe886bb3e72c557741a 2024-12-02T05:18:16,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/0039dedad5f3473fb9488d0de2979c9e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/0039dedad5f3473fb9488d0de2979c9e 2024-12-02T05:18:16,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/36d39dc3bd074f72b4284578a1db3b7c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/36d39dc3bd074f72b4284578a1db3b7c 2024-12-02T05:18:16,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/bfa83170b0ff4bac9bfbf4d0e9dcf37a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/bfa83170b0ff4bac9bfbf4d0e9dcf37a 2024-12-02T05:18:16,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/12a9249279754d49bd2107749818bd8d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/12a9249279754d49bd2107749818bd8d 2024-12-02T05:18:16,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/7d63f8ebdf9c46ee8832fe1093e97062 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/7d63f8ebdf9c46ee8832fe1093e97062 2024-12-02T05:18:16,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/9ad0bf9b1a974b1399667b4ef6e9adb3 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/9ad0bf9b1a974b1399667b4ef6e9adb3 2024-12-02T05:18:16,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/e1d9ff3fd91f46708aa495903c7e8bc9 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/e1d9ff3fd91f46708aa495903c7e8bc9 2024-12-02T05:18:16,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/2ac407fd61ab44e0b6a13211ec33cfb9 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/2ac407fd61ab44e0b6a13211ec33cfb9 2024-12-02T05:18:16,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/17879430241d4f2185a19dabc2bf67ae to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/17879430241d4f2185a19dabc2bf67ae 2024-12-02T05:18:16,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/1fbabd9be978403499dc240515fcabfb to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/1fbabd9be978403499dc240515fcabfb 2024-12-02T05:18:16,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/f9255bf28b414bb09eea70599d487f9e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/f9255bf28b414bb09eea70599d487f9e 2024-12-02T05:18:16,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/6a8dfdc9d50e40b1a40cd9fa86aebbdb to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/6a8dfdc9d50e40b1a40cd9fa86aebbdb 2024-12-02T05:18:16,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/d132c358d0ea4ebe8e0b3b8996028895 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/d132c358d0ea4ebe8e0b3b8996028895 2024-12-02T05:18:16,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/18d8a88f1274443a81b231aa277d249c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/18d8a88f1274443a81b231aa277d249c 2024-12-02T05:18:16,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/23fcb59339a449168546a19cae7307f6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/23fcb59339a449168546a19cae7307f6 2024-12-02T05:18:16,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5c59212df413455bb8958500cede782f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/821d8021ff4a4802a867dd4488fd53be, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/c6dcd3ce68954736b2d25b6f070501f0, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/950968bbc92248e88c074e0b40762ec3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/c9c32e33233c4615bb2c75786d9675f5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/8dd39b7153f344778ecd463098b45d6a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/7b042a07248d43bfbf94b9331ffb41ed, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/b0203495d92a4610b9af71c56f64dfce, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/d04e18e9ce2c409d85a30b3d3a2178cd, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/438335314d744dd38a3060f0674c7d9e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5f9121cfd9654631a33aa5021e9441cd, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/09c1d122f31a4673a8fae16b5e9e0eca, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/aeba02159c144544b646e843fbd3a9cd, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/367be8ae7812407c814c9594881035f2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2ecca740e11949d59992147bacdf3207, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/a3581974ef1a427f90e086feb86d4f41, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/8aa01cfffac64b26a14b4b8167c4a5b8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/287e17915fc54f548cd3cda159b1e2c8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3926efffed7a4f8487d3881260b58615, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2d4b5c0e8e304f0d986826aadc17a6c6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3364b619e05f4411989ec477e26dd9a4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/b3bec0740452489b832223a72e04e771, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/fa2663f204674b3cab1c9d17edeccce8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3864d6e885434021aa55adec7235246c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2c548e0e9035488282b8443ee1d83bd4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5079ea5267de47dea2e777368a43f935, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/a728f3265ac6402da656c39702ae56b3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/ba62c50045744f3192028dd1228153e2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/eb1f64285bae442aa3ae203b86c725f7, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/bc3c49ce835047dabedda2eb8c2a433c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/0584b026f3d14f869a57da678bd36744] to archive 2024-12-02T05:18:16,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:18:16,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5c59212df413455bb8958500cede782f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5c59212df413455bb8958500cede782f 2024-12-02T05:18:16,320 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/821d8021ff4a4802a867dd4488fd53be to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/821d8021ff4a4802a867dd4488fd53be 2024-12-02T05:18:16,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/c6dcd3ce68954736b2d25b6f070501f0 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/c6dcd3ce68954736b2d25b6f070501f0 2024-12-02T05:18:16,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/950968bbc92248e88c074e0b40762ec3 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/950968bbc92248e88c074e0b40762ec3 2024-12-02T05:18:16,323 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/c9c32e33233c4615bb2c75786d9675f5 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/c9c32e33233c4615bb2c75786d9675f5 2024-12-02T05:18:16,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/8dd39b7153f344778ecd463098b45d6a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/8dd39b7153f344778ecd463098b45d6a 2024-12-02T05:18:16,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/7b042a07248d43bfbf94b9331ffb41ed to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/7b042a07248d43bfbf94b9331ffb41ed 2024-12-02T05:18:16,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/b0203495d92a4610b9af71c56f64dfce to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/b0203495d92a4610b9af71c56f64dfce 2024-12-02T05:18:16,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/d04e18e9ce2c409d85a30b3d3a2178cd to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/d04e18e9ce2c409d85a30b3d3a2178cd 2024-12-02T05:18:16,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/438335314d744dd38a3060f0674c7d9e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/438335314d744dd38a3060f0674c7d9e 2024-12-02T05:18:16,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5f9121cfd9654631a33aa5021e9441cd to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5f9121cfd9654631a33aa5021e9441cd 2024-12-02T05:18:16,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/09c1d122f31a4673a8fae16b5e9e0eca to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/09c1d122f31a4673a8fae16b5e9e0eca 2024-12-02T05:18:16,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/aeba02159c144544b646e843fbd3a9cd to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/aeba02159c144544b646e843fbd3a9cd 2024-12-02T05:18:16,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/367be8ae7812407c814c9594881035f2 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/367be8ae7812407c814c9594881035f2 2024-12-02T05:18:16,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2ecca740e11949d59992147bacdf3207 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2ecca740e11949d59992147bacdf3207 2024-12-02T05:18:16,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/a3581974ef1a427f90e086feb86d4f41 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/a3581974ef1a427f90e086feb86d4f41 2024-12-02T05:18:16,332 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/8aa01cfffac64b26a14b4b8167c4a5b8 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/8aa01cfffac64b26a14b4b8167c4a5b8 2024-12-02T05:18:16,332 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/287e17915fc54f548cd3cda159b1e2c8 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/287e17915fc54f548cd3cda159b1e2c8 2024-12-02T05:18:16,333 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3926efffed7a4f8487d3881260b58615 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3926efffed7a4f8487d3881260b58615 2024-12-02T05:18:16,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2d4b5c0e8e304f0d986826aadc17a6c6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2d4b5c0e8e304f0d986826aadc17a6c6 2024-12-02T05:18:16,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3364b619e05f4411989ec477e26dd9a4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3364b619e05f4411989ec477e26dd9a4 2024-12-02T05:18:16,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/b3bec0740452489b832223a72e04e771 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/b3bec0740452489b832223a72e04e771 2024-12-02T05:18:16,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/fa2663f204674b3cab1c9d17edeccce8 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/fa2663f204674b3cab1c9d17edeccce8 2024-12-02T05:18:16,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3864d6e885434021aa55adec7235246c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/3864d6e885434021aa55adec7235246c 2024-12-02T05:18:16,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2c548e0e9035488282b8443ee1d83bd4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/2c548e0e9035488282b8443ee1d83bd4 2024-12-02T05:18:16,338 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5079ea5267de47dea2e777368a43f935 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/5079ea5267de47dea2e777368a43f935 2024-12-02T05:18:16,338 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/a728f3265ac6402da656c39702ae56b3 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/a728f3265ac6402da656c39702ae56b3 2024-12-02T05:18:16,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/ba62c50045744f3192028dd1228153e2 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/ba62c50045744f3192028dd1228153e2 2024-12-02T05:18:16,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/eb1f64285bae442aa3ae203b86c725f7 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/eb1f64285bae442aa3ae203b86c725f7 2024-12-02T05:18:16,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/bc3c49ce835047dabedda2eb8c2a433c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/bc3c49ce835047dabedda2eb8c2a433c 2024-12-02T05:18:16,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/0584b026f3d14f869a57da678bd36744 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/0584b026f3d14f869a57da678bd36744 2024-12-02T05:18:16,345 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/recovered.edits/497.seqid, newMaxSeqId=497, maxSeqId=1 2024-12-02T05:18:16,346 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. 2024-12-02T05:18:16,346 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 2bec4a5721d5a74dad6f846b3202c90f: Waiting for close lock at 1733116695010Running coprocessor pre-close hooks at 1733116695010Disabling compacts and flushes for region at 1733116695010Disabling writes for close at 1733116695010Obtaining lock to block concurrent updates at 1733116695010Preparing flush snapshotting stores in 2bec4a5721d5a74dad6f846b3202c90f at 1733116695010Finished memstore snapshotting TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733116695010Flushing stores of TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f. at 1733116695011 (+1 ms)Flushing 2bec4a5721d5a74dad6f846b3202c90f/A: creating writer at 1733116695011Flushing 2bec4a5721d5a74dad6f846b3202c90f/A: appending metadata at 1733116695033 (+22 ms)Flushing 2bec4a5721d5a74dad6f846b3202c90f/A: closing flushed file at 1733116695033Flushing 2bec4a5721d5a74dad6f846b3202c90f/B: creating writer at 1733116695811 (+778 ms)Flushing 2bec4a5721d5a74dad6f846b3202c90f/B: appending metadata at 1733116695815 (+4 ms)Flushing 2bec4a5721d5a74dad6f846b3202c90f/B: closing flushed file at 1733116695815Flushing 2bec4a5721d5a74dad6f846b3202c90f/C: creating writer at 1733116696228 (+413 ms)Flushing 2bec4a5721d5a74dad6f846b3202c90f/C: appending metadata at 1733116696233 (+5 ms)Flushing 2bec4a5721d5a74dad6f846b3202c90f/C: closing flushed file at 1733116696233Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dcc0ee9: reopening flushed file at 1733116696245 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20f17b45: reopening flushed file at 1733116696251 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e23904f: reopening flushed file at 1733116696254 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2bec4a5721d5a74dad6f846b3202c90f in 1248ms, sequenceid=494, compaction requested=true at 1733116696258 (+4 ms)Writing region close event to WAL at 1733116696343 (+85 ms)Running coprocessor post-close hooks at 1733116696346 (+3 ms)Closed at 1733116696346 2024-12-02T05:18:16,347 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:18:16,348 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=2bec4a5721d5a74dad6f846b3202c90f, regionState=CLOSED 2024-12-02T05:18:16,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:18:16,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-02T05:18:16,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 2bec4a5721d5a74dad6f846b3202c90f, server=f49461030c22,46495,1733116594459 in 1.4930 sec 2024-12-02T05:18:16,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-02T05:18:16,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2bec4a5721d5a74dad6f846b3202c90f, UNASSIGN in 1.4980 sec 2024-12-02T05:18:16,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-02T05:18:16,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5030 sec 2024-12-02T05:18:16,356 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116696356"}]},"ts":"1733116696356"} 2024-12-02T05:18:16,357 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T05:18:16,357 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T05:18:16,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5140 sec 2024-12-02T05:18:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T05:18:16,998 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:16,999 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T05:18:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:17,002 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-02T05:18:17,003 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:17,008 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:18:17,011 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/recovered.edits] 2024-12-02T05:18:17,014 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0a03c3be12dc479cb194e233a92dd8e7 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/0a03c3be12dc479cb194e233a92dd8e7 2024-12-02T05:18:17,016 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/982174d4d15d41a99bc7671501b05189 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/982174d4d15d41a99bc7671501b05189 2024-12-02T05:18:17,017 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/e7d1ba325d23496daeeb09a5c6707dee to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/e7d1ba325d23496daeeb09a5c6707dee 2024-12-02T05:18:17,019 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/f17a6f302e0a41a98b26a71ea7acc85c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/A/f17a6f302e0a41a98b26a71ea7acc85c 2024-12-02T05:18:17,023 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/4287929ad00b40398ad383fa09f75fc4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/4287929ad00b40398ad383fa09f75fc4 2024-12-02T05:18:17,024 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/5b1cb019c8304e83bbdd05b2cda5afc1 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/5b1cb019c8304e83bbdd05b2cda5afc1 2024-12-02T05:18:17,026 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/c5d5392aad8841f784fb73764c9be895 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/c5d5392aad8841f784fb73764c9be895 2024-12-02T05:18:17,027 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/d5031e7f49164f10a4f383963ba9fdce to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/B/d5031e7f49164f10a4f383963ba9fdce 2024-12-02T05:18:17,030 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/1099a2a3432b45c6949d72e1e7db30c6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/1099a2a3432b45c6949d72e1e7db30c6 2024-12-02T05:18:17,031 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/13a37c9e87584c63aaf8fc50898765b9 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/13a37c9e87584c63aaf8fc50898765b9 2024-12-02T05:18:17,033 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/63f847f23c7a4e20b7fe7f990b2543ba to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/63f847f23c7a4e20b7fe7f990b2543ba 2024-12-02T05:18:17,034 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/7c8b95b33e9f4e72b70e5d0e7bda3491 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/C/7c8b95b33e9f4e72b70e5d0e7bda3491 2024-12-02T05:18:17,038 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/recovered.edits/497.seqid to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f/recovered.edits/497.seqid 2024-12-02T05:18:17,038 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/2bec4a5721d5a74dad6f846b3202c90f 2024-12-02T05:18:17,038 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T05:18:17,041 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:17,044 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T05:18:17,047 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T05:18:17,048 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:17,048 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T05:18:17,048 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733116697048"}]},"ts":"9223372036854775807"} 2024-12-02T05:18:17,050 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T05:18:17,050 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2bec4a5721d5a74dad6f846b3202c90f, NAME => 'TestAcidGuarantees,,1733116664244.2bec4a5721d5a74dad6f846b3202c90f.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T05:18:17,050 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T05:18:17,051 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733116697050"}]},"ts":"9223372036854775807"} 2024-12-02T05:18:17,053 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T05:18:17,053 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:17,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 54 msec 2024-12-02T05:18:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-02T05:18:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T05:18:17,108 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:17,115 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 249), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=574 (was 568) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6361 (was 6927) 2024-12-02T05:18:17,124 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=574, ProcessCount=11, AvailableMemoryMB=6361 2024-12-02T05:18:17,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T05:18:17,125 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T05:18:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:17,127 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T05:18:17,127 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:17,127 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-12-02T05:18:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T05:18:17,128 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T05:18:17,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742147_1323 (size=1061) 2024-12-02T05:18:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T05:18:17,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T05:18:17,535 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c 2024-12-02T05:18:17,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742148_1324 (size=53) 2024-12-02T05:18:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T05:18:17,941 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:18:17,941 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ff61f31505788790cd78022602747873, disabling compactions & flushes 2024-12-02T05:18:17,941 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:17,941 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:17,941 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. after waiting 0 ms 2024-12-02T05:18:17,941 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:17,941 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:17,941 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ff61f31505788790cd78022602747873: Waiting for close lock at 1733116697941Disabling compacts and flushes for region at 1733116697941Disabling writes for close at 1733116697941Writing region close event to WAL at 1733116697941Closed at 1733116697941 2024-12-02T05:18:17,942 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T05:18:17,943 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733116697942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733116697942"}]},"ts":"1733116697942"} 2024-12-02T05:18:17,944 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T05:18:17,946 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T05:18:17,946 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116697946"}]},"ts":"1733116697946"} 2024-12-02T05:18:17,948 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T05:18:17,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff61f31505788790cd78022602747873, ASSIGN}] 2024-12-02T05:18:17,950 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff61f31505788790cd78022602747873, ASSIGN 2024-12-02T05:18:17,951 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff61f31505788790cd78022602747873, ASSIGN; state=OFFLINE, location=f49461030c22,46495,1733116594459; forceNewPlan=false, retain=false 2024-12-02T05:18:18,102 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=ff61f31505788790cd78022602747873, regionState=OPENING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:18:18,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff61f31505788790cd78022602747873, ASSIGN because future has completed 2024-12-02T05:18:18,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:18:18,259 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:18,259 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} 2024-12-02T05:18:18,259 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ff61f31505788790cd78022602747873 2024-12-02T05:18:18,259 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:18:18,259 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for ff61f31505788790cd78022602747873 2024-12-02T05:18:18,259 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for ff61f31505788790cd78022602747873 2024-12-02T05:18:18,261 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ff61f31505788790cd78022602747873 2024-12-02T05:18:18,262 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:18:18,262 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff61f31505788790cd78022602747873 columnFamilyName A 2024-12-02T05:18:18,262 DEBUG [StoreOpener-ff61f31505788790cd78022602747873-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:18,262 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.HStore(327): Store=ff61f31505788790cd78022602747873/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:18:18,263 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ff61f31505788790cd78022602747873 2024-12-02T05:18:18,264 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:18:18,264 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff61f31505788790cd78022602747873 columnFamilyName B 2024-12-02T05:18:18,264 DEBUG [StoreOpener-ff61f31505788790cd78022602747873-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:18,264 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.HStore(327): Store=ff61f31505788790cd78022602747873/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:18:18,264 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ff61f31505788790cd78022602747873 2024-12-02T05:18:18,265 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:18:18,265 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff61f31505788790cd78022602747873 columnFamilyName C 2024-12-02T05:18:18,265 DEBUG [StoreOpener-ff61f31505788790cd78022602747873-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:18,265 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.HStore(327): Store=ff61f31505788790cd78022602747873/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:18:18,266 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:18,266 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for ff61f31505788790cd78022602747873 2024-12-02T05:18:18,266 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873 2024-12-02T05:18:18,266 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873 2024-12-02T05:18:18,267 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for ff61f31505788790cd78022602747873 2024-12-02T05:18:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T05:18:18,267 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for ff61f31505788790cd78022602747873 2024-12-02T05:18:18,268 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T05:18:18,269 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for ff61f31505788790cd78022602747873 2024-12-02T05:18:18,270 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T05:18:18,270 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened ff61f31505788790cd78022602747873; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66125271, jitterRate=-0.014656677842140198}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T05:18:18,271 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff61f31505788790cd78022602747873 2024-12-02T05:18:18,271 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for ff61f31505788790cd78022602747873: Running coprocessor pre-open hook at 1733116698259Writing region info on filesystem at 1733116698260 (+1 ms)Initializing all the Stores at 1733116698260Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116698260Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116698261 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116698261Cleaning up temporary data from old regions at 1733116698267 (+6 ms)Running coprocessor post-open hooks at 1733116698271 (+4 ms)Region opened successfully at 1733116698271 2024-12-02T05:18:18,272 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., pid=93, masterSystemTime=1733116698256 2024-12-02T05:18:18,273 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:18,273 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:18,274 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=ff61f31505788790cd78022602747873, regionState=OPEN, openSeqNum=2, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:18:18,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:18:18,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-02T05:18:18,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 in 172 msec 2024-12-02T05:18:18,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-02T05:18:18,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff61f31505788790cd78022602747873, ASSIGN in 329 msec 2024-12-02T05:18:18,280 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T05:18:18,280 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116698280"}]},"ts":"1733116698280"} 2024-12-02T05:18:18,281 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T05:18:18,282 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T05:18:18,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1570 sec 2024-12-02T05:18:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T05:18:19,279 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:19,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b5885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:19,280 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:19,281 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:19,282 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52022124, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:19,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:19,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:19,285 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38cc1736, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:19,287 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:19,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:19,291 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T05:18:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f49461030c22,42731,1733116593060 2024-12-02T05:18:19,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@213e1ed1 2024-12-02T05:18:19,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T05:18:19,293 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T05:18:19,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T05:18:19,294 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T05:18:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:19,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742149_1325 (size=1097) 2024-12-02T05:18:19,707 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T05:18:19,707 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T05:18:19,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T05:18:19,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff61f31505788790cd78022602747873, REOPEN/MOVE}] 2024-12-02T05:18:19,723 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff61f31505788790cd78022602747873, REOPEN/MOVE 2024-12-02T05:18:19,740 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=ff61f31505788790cd78022602747873, regionState=CLOSING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:18:19,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff61f31505788790cd78022602747873, REOPEN/MOVE because future has completed 2024-12-02T05:18:19,743 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T05:18:19,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:18:19,897 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close ff61f31505788790cd78022602747873 2024-12-02T05:18:19,897 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T05:18:19,897 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing ff61f31505788790cd78022602747873, disabling compactions & flushes 2024-12-02T05:18:19,897 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:19,897 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:19,897 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. after waiting 0 ms 2024-12-02T05:18:19,897 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:19,908 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T05:18:19,909 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:19,909 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for ff61f31505788790cd78022602747873: Waiting for close lock at 1733116699897Running coprocessor pre-close hooks at 1733116699897Disabling compacts and flushes for region at 1733116699897Disabling writes for close at 1733116699897Writing region close event to WAL at 1733116699904 (+7 ms)Running coprocessor post-close hooks at 1733116699909 (+5 ms)Closed at 1733116699909 2024-12-02T05:18:19,909 WARN [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: ff61f31505788790cd78022602747873 to self. 2024-12-02T05:18:19,912 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed ff61f31505788790cd78022602747873 2024-12-02T05:18:19,914 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=ff61f31505788790cd78022602747873, regionState=CLOSED 2024-12-02T05:18:19,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:18:19,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-12-02T05:18:19,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 in 174 msec 2024-12-02T05:18:19,921 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff61f31505788790cd78022602747873, REOPEN/MOVE; state=CLOSED, location=f49461030c22,46495,1733116594459; forceNewPlan=false, retain=true 2024-12-02T05:18:20,071 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=ff61f31505788790cd78022602747873, regionState=OPENING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff61f31505788790cd78022602747873, REOPEN/MOVE because future has completed 2024-12-02T05:18:20,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:18:20,229 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:20,229 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} 2024-12-02T05:18:20,230 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ff61f31505788790cd78022602747873 2024-12-02T05:18:20,230 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:18:20,230 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for ff61f31505788790cd78022602747873 2024-12-02T05:18:20,230 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for ff61f31505788790cd78022602747873 2024-12-02T05:18:20,231 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ff61f31505788790cd78022602747873 2024-12-02T05:18:20,232 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:18:20,233 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff61f31505788790cd78022602747873 columnFamilyName A 2024-12-02T05:18:20,234 DEBUG [StoreOpener-ff61f31505788790cd78022602747873-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:20,234 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.HStore(327): Store=ff61f31505788790cd78022602747873/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:18:20,235 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ff61f31505788790cd78022602747873 2024-12-02T05:18:20,235 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:18:20,236 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff61f31505788790cd78022602747873 columnFamilyName B 2024-12-02T05:18:20,236 DEBUG [StoreOpener-ff61f31505788790cd78022602747873-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:20,236 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.HStore(327): Store=ff61f31505788790cd78022602747873/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:18:20,236 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ff61f31505788790cd78022602747873 2024-12-02T05:18:20,237 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:18:20,237 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff61f31505788790cd78022602747873 columnFamilyName C 2024-12-02T05:18:20,237 DEBUG [StoreOpener-ff61f31505788790cd78022602747873-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:20,237 INFO [StoreOpener-ff61f31505788790cd78022602747873-1 {}] regionserver.HStore(327): Store=ff61f31505788790cd78022602747873/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:18:20,238 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:20,238 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for ff61f31505788790cd78022602747873 2024-12-02T05:18:20,238 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873 2024-12-02T05:18:20,239 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873 2024-12-02T05:18:20,240 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for ff61f31505788790cd78022602747873 2024-12-02T05:18:20,240 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for ff61f31505788790cd78022602747873 2024-12-02T05:18:20,240 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T05:18:20,241 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for ff61f31505788790cd78022602747873 2024-12-02T05:18:20,241 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened ff61f31505788790cd78022602747873; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66320655, jitterRate=-0.011745229363441467}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T05:18:20,241 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff61f31505788790cd78022602747873 2024-12-02T05:18:20,242 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for ff61f31505788790cd78022602747873: Running coprocessor pre-open hook at 1733116700230Writing region info on filesystem at 1733116700230Initializing all the Stores at 1733116700231 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116700231Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116700231Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116700231Cleaning up temporary data from old regions at 1733116700240 (+9 ms)Running coprocessor post-open hooks at 1733116700241 (+1 ms)Region opened successfully at 1733116700242 (+1 ms) 2024-12-02T05:18:20,242 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., pid=98, masterSystemTime=1733116700226 2024-12-02T05:18:20,244 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:20,244 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:20,244 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=ff61f31505788790cd78022602747873, regionState=OPEN, openSeqNum=5, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:18:20,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-12-02T05:18:20,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 in 172 msec 2024-12-02T05:18:20,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-02T05:18:20,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff61f31505788790cd78022602747873, REOPEN/MOVE in 529 msec 2024-12-02T05:18:20,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-02T05:18:20,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 536 msec 2024-12-02T05:18:20,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 955 msec 2024-12-02T05:18:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-12-02T05:18:20,252 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:20,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,253 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:20,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:20,254 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:20,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:20,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:20,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36436c63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:20,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:20,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:20,255 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31ea68e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,257 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:20,257 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:20,257 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13a93c7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:20,259 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7072f953, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:20,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:20,261 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf41bbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:20,263 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e37d8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:20,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:20,265 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:20,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:20,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:20,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0858bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:20,267 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c1168cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:20,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:20,269 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab1568b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:20,270 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:20,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a41501f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,272 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:20,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:20,273 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e033a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:20,274 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64369193, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,276 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:20,276 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:20,276 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43c1783e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:20,278 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7445db58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:20,279 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:20,280 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79cc4e7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:20,281 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc06cd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:20,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:20,282 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac20fb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:20,284 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:20,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23f010d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b779047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:20,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:20,286 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59bbf57d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:20,287 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:20,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39ad862a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:20,288 DEBUG [Thread-1500 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:20,288 DEBUG [Thread-1513 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:20,288 DEBUG [Thread-1511 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:20,288 DEBUG [Thread-1502 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:20,288 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:18:20,289 DEBUG [Thread-1505 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:20,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-02T05:18:20,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:20,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:20,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T05:18:20,291 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:20,291 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:20,291 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:18:20,292 DEBUG [Thread-1509 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:20,292 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:18:20,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:18:20,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:18:20,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:18:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:18:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:18:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:18:20,297 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:18:20,300 DEBUG [Thread-1507 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:20,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:20,302 DEBUG [Thread-1498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:20,303 DEBUG [Thread-1494 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:18:20,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:20,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:20,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T05:18:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:18:20,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:18:20,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:20,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:20,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:20,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:20,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:20,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:20,312 DEBUG [Thread-1496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:20,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:20,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:20,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:20,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:18:20,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020c57c6086cc5450dbbf1a38887ecd0e4_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116700302/Put/seqid=0 2024-12-02T05:18:20,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742150_1326 (size=14594) 2024-12-02T05:18:20,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733116700328, max timestamp=1733116700342Num uniques 859; 2024-12-02T05:18:20,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T05:18:20,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733116700328, max timestamp=1733116700342Num uniques 859; 2024-12-02T05:18:20,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T05:18:20,345 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:20,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733116700328, max timestamp=1733116700342Num uniques 859; 2024-12-02T05:18:20,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T05:18:20,349 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020c57c6086cc5450dbbf1a38887ecd0e4_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020c57c6086cc5450dbbf1a38887ecd0e4_ff61f31505788790cd78022602747873 2024-12-02T05:18:20,350 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/af99528764f945fdac0a6e11e7a1ce1f, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:20,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/af99528764f945fdac0a6e11e7a1ce1f is 175, key is test_row_0/A:col10/1733116700302/Put/seqid=0 2024-12-02T05:18:20,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742151_1327 (size=39549) 2024-12-02T05:18:20,356 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/af99528764f945fdac0a6e11e7a1ce1f 2024-12-02T05:18:20,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116760370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/fb6d0e52f41b48fb8035573ea6ded859 is 50, key is test_row_0/B:col10/1733116700302/Put/seqid=0 2024-12-02T05:18:20,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116760371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116760371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116760372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116760382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T05:18:20,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742152_1328 (size=12001) 2024-12-02T05:18:20,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/fb6d0e52f41b48fb8035573ea6ded859 2024-12-02T05:18:20,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/c6e06f88fc01408380bd72c304e8ed66 is 50, key is test_row_0/C:col10/1733116700302/Put/seqid=0 2024-12-02T05:18:20,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742153_1329 (size=12001) 2024-12-02T05:18:20,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/c6e06f88fc01408380bd72c304e8ed66 2024-12-02T05:18:20,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/af99528764f945fdac0a6e11e7a1ce1f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/af99528764f945fdac0a6e11e7a1ce1f 2024-12-02T05:18:20,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/af99528764f945fdac0a6e11e7a1ce1f, entries=200, sequenceid=15, filesize=38.6 K 2024-12-02T05:18:20,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/fb6d0e52f41b48fb8035573ea6ded859 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/fb6d0e52f41b48fb8035573ea6ded859 2024-12-02T05:18:20,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T05:18:20,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:20,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:20,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:20,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:20,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:20,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/fb6d0e52f41b48fb8035573ea6ded859, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T05:18:20,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/c6e06f88fc01408380bd72c304e8ed66 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/c6e06f88fc01408380bd72c304e8ed66 2024-12-02T05:18:20,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/c6e06f88fc01408380bd72c304e8ed66, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T05:18:20,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for ff61f31505788790cd78022602747873 in 157ms, sequenceid=15, compaction requested=false 2024-12-02T05:18:20,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:20,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-02T05:18:20,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:20,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:20,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:20,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:20,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:20,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:20,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e3831dc138614da9b1ef38ae4f0563e3_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116700370/Put/seqid=0 2024-12-02T05:18:20,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742154_1330 (size=14594) 2024-12-02T05:18:20,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116760498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116760505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116760506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116760506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116760506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T05:18:20,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:20,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:20,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:20,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:20,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T05:18:20,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116760618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116760618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116760618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116760627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116760717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T05:18:20,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:20,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:20,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:20,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:20,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:20,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116760827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116760827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116760837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116760837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:20,903 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:20,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T05:18:20,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:20,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:20,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:20,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:20,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:20,908 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e3831dc138614da9b1ef38ae4f0563e3_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e3831dc138614da9b1ef38ae4f0563e3_ff61f31505788790cd78022602747873 2024-12-02T05:18:20,908 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/2f6a5ea2267046bc92c7af2d1786920d, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:20,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/2f6a5ea2267046bc92c7af2d1786920d is 175, key is test_row_0/A:col10/1733116700370/Put/seqid=0 2024-12-02T05:18:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:20,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742155_1331 (size=39549) 2024-12-02T05:18:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T05:18:21,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116761027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:21,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T05:18:21,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:21,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:21,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:21,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:21,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:21,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116761137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116761137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:21,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116761147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116761148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:21,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T05:18:21,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:21,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:21,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:21,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:21,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:21,315 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/2f6a5ea2267046bc92c7af2d1786920d 2024-12-02T05:18:21,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/0895b10665df4a7686c2068c4eb519d9 is 50, key is test_row_0/B:col10/1733116700370/Put/seqid=0 2024-12-02T05:18:21,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742156_1332 (size=12001) 2024-12-02T05:18:21,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/0895b10665df4a7686c2068c4eb519d9 2024-12-02T05:18:21,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/8bb31c5dc8074a878feeed0a77a3162e is 50, key is test_row_0/C:col10/1733116700370/Put/seqid=0 2024-12-02T05:18:21,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742157_1333 (size=12001) 2024-12-02T05:18:21,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/8bb31c5dc8074a878feeed0a77a3162e 2024-12-02T05:18:21,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/2f6a5ea2267046bc92c7af2d1786920d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2f6a5ea2267046bc92c7af2d1786920d 2024-12-02T05:18:21,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T05:18:21,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:21,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:21,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:21,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:21,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:21,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2f6a5ea2267046bc92c7af2d1786920d, entries=200, sequenceid=50, filesize=38.6 K 2024-12-02T05:18:21,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/0895b10665df4a7686c2068c4eb519d9 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/0895b10665df4a7686c2068c4eb519d9 2024-12-02T05:18:21,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/0895b10665df4a7686c2068c4eb519d9, entries=150, sequenceid=50, filesize=11.7 K 2024-12-02T05:18:21,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/8bb31c5dc8074a878feeed0a77a3162e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/8bb31c5dc8074a878feeed0a77a3162e 2024-12-02T05:18:21,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/8bb31c5dc8074a878feeed0a77a3162e, entries=150, sequenceid=50, filesize=11.7 K 2024-12-02T05:18:21,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for ff61f31505788790cd78022602747873 in 893ms, sequenceid=50, compaction requested=false 2024-12-02T05:18:21,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T05:18:21,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T05:18:21,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:21,524 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T05:18:21,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:21,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:21,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:21,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:21,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:21,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:21,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120245845108a2a1457fb87cd490e5a822ac_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116700504/Put/seqid=0 2024-12-02T05:18:21,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742158_1334 (size=12154) 2024-12-02T05:18:21,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:21,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:21,544 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120245845108a2a1457fb87cd490e5a822ac_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120245845108a2a1457fb87cd490e5a822ac_ff61f31505788790cd78022602747873 2024-12-02T05:18:21,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/4230e624f4a64e55bd0e2697feff5c35, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:21,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/4230e624f4a64e55bd0e2697feff5c35 is 175, key is test_row_0/A:col10/1733116700504/Put/seqid=0 2024-12-02T05:18:21,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742159_1335 (size=30955) 2024-12-02T05:18:21,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116701605, max timestamp=1733116701633Num uniques 859; 2024-12-02T05:18:21,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:18:21,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116701605, max timestamp=1733116701633Num uniques 859; 2024-12-02T05:18:21,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116701605, max timestamp=1733116701633Num uniques 859; 2024-12-02T05:18:21,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:18:21,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:18:21,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116761683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:21,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116761684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:21,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116761684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:21,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116761684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:21,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116761686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:21,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116761797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:21,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116761797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:21,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116761797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:21,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116761798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:21,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116761807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:21,875 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T05:18:21,969 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/4230e624f4a64e55bd0e2697feff5c35 2024-12-02T05:18:21,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/985151927d674c90a95e1cb6ee06aa58 is 50, key is test_row_0/B:col10/1733116700504/Put/seqid=0 2024-12-02T05:18:21,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742160_1336 (size=12001) 2024-12-02T05:18:22,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116762007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116762017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116762018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116762018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116762018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116762317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116762327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116762327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116762328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116762328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,382 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/985151927d674c90a95e1cb6ee06aa58 2024-12-02T05:18:22,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/22a7f32513a648109f3412281bd16da4 is 50, key is test_row_0/C:col10/1733116700504/Put/seqid=0 2024-12-02T05:18:22,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742161_1337 (size=12001) 2024-12-02T05:18:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T05:18:22,793 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/22a7f32513a648109f3412281bd16da4 2024-12-02T05:18:22,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/4230e624f4a64e55bd0e2697feff5c35 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/4230e624f4a64e55bd0e2697feff5c35 2024-12-02T05:18:22,801 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/4230e624f4a64e55bd0e2697feff5c35, entries=150, sequenceid=60, filesize=30.2 K 2024-12-02T05:18:22,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/985151927d674c90a95e1cb6ee06aa58 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/985151927d674c90a95e1cb6ee06aa58 2024-12-02T05:18:22,805 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/985151927d674c90a95e1cb6ee06aa58, entries=150, sequenceid=60, filesize=11.7 K 2024-12-02T05:18:22,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/22a7f32513a648109f3412281bd16da4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/22a7f32513a648109f3412281bd16da4 2024-12-02T05:18:22,809 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/22a7f32513a648109f3412281bd16da4, entries=150, sequenceid=60, filesize=11.7 K 2024-12-02T05:18:22,810 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ff61f31505788790cd78022602747873 in 1286ms, sequenceid=60, compaction requested=true 2024-12-02T05:18:22,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:22,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:22,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-02T05:18:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-02T05:18:22,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-02T05:18:22,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5190 sec 2024-12-02T05:18:22,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.5250 sec 2024-12-02T05:18:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:22,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-02T05:18:22,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:22,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:22,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:22,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:22,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:22,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:22,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bf3c7ea75c0444e1ac52b113c1b0b48d_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116701684/Put/seqid=0 2024-12-02T05:18:22,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742162_1338 (size=14594) 2024-12-02T05:18:22,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116762858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116762859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116762860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116762861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116762861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116762977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116762977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116762977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116762977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:22,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116762977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:22,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:22,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116763187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116763187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:23,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116763187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:23,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116763198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:23,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:23,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116763198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:23,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:23,258 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:23,263 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bf3c7ea75c0444e1ac52b113c1b0b48d_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bf3c7ea75c0444e1ac52b113c1b0b48d_ff61f31505788790cd78022602747873 2024-12-02T05:18:23,264 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/2e77057d7cf748e0bdc5e2137ff50b0f, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:23,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/2e77057d7cf748e0bdc5e2137ff50b0f is 175, key is test_row_0/A:col10/1733116701684/Put/seqid=0 2024-12-02T05:18:23,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742163_1339 (size=39549) 2024-12-02T05:18:23,272 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/2e77057d7cf748e0bdc5e2137ff50b0f 2024-12-02T05:18:23,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/82ca838021c04257b697b19f0d34e207 is 50, key is test_row_0/B:col10/1733116701684/Put/seqid=0 2024-12-02T05:18:23,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742164_1340 (size=12001) 2024-12-02T05:18:23,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/82ca838021c04257b697b19f0d34e207 2024-12-02T05:18:23,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/82524a04db434915ab00ac6215f5a4f4 is 50, key is test_row_0/C:col10/1733116701684/Put/seqid=0 2024-12-02T05:18:23,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742165_1341 (size=12001) 2024-12-02T05:18:23,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116763507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:23,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116763508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:23,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116763508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:23,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:23,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116763508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:23,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116763508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:23,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/82524a04db434915ab00ac6215f5a4f4 2024-12-02T05:18:23,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/2e77057d7cf748e0bdc5e2137ff50b0f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2e77057d7cf748e0bdc5e2137ff50b0f 2024-12-02T05:18:23,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2e77057d7cf748e0bdc5e2137ff50b0f, entries=200, sequenceid=97, filesize=38.6 K 2024-12-02T05:18:23,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/82ca838021c04257b697b19f0d34e207 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/82ca838021c04257b697b19f0d34e207 2024-12-02T05:18:23,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/82ca838021c04257b697b19f0d34e207, entries=150, sequenceid=97, filesize=11.7 K 2024-12-02T05:18:23,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/82524a04db434915ab00ac6215f5a4f4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/82524a04db434915ab00ac6215f5a4f4 2024-12-02T05:18:23,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/82524a04db434915ab00ac6215f5a4f4, entries=150, sequenceid=97, filesize=11.7 K 2024-12-02T05:18:23,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for ff61f31505788790cd78022602747873 in 900ms, sequenceid=97, compaction requested=true 2024-12-02T05:18:23,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:23,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:18:23,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:23,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:18:23,747 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:18:23,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:23,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:18:23,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T05:18:23,748 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:18:23,749 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:18:23,749 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/B is initiating minor compaction (all files) 2024-12-02T05:18:23,749 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/B in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:23,749 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/fb6d0e52f41b48fb8035573ea6ded859, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/0895b10665df4a7686c2068c4eb519d9, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/985151927d674c90a95e1cb6ee06aa58, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/82ca838021c04257b697b19f0d34e207] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=46.9 K 2024-12-02T05:18:23,751 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149602 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:18:23,751 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/A is initiating minor compaction (all files) 2024-12-02T05:18:23,751 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting fb6d0e52f41b48fb8035573ea6ded859, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733116700296 2024-12-02T05:18:23,751 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/A in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:23,751 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/af99528764f945fdac0a6e11e7a1ce1f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2f6a5ea2267046bc92c7af2d1786920d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/4230e624f4a64e55bd0e2697feff5c35, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2e77057d7cf748e0bdc5e2137ff50b0f] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=146.1 K 2024-12-02T05:18:23,751 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:23,752 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 0895b10665df4a7686c2068c4eb519d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733116700357 2024-12-02T05:18:23,752 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/af99528764f945fdac0a6e11e7a1ce1f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2f6a5ea2267046bc92c7af2d1786920d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/4230e624f4a64e55bd0e2697feff5c35, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2e77057d7cf748e0bdc5e2137ff50b0f] 2024-12-02T05:18:23,752 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 985151927d674c90a95e1cb6ee06aa58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733116700494 2024-12-02T05:18:23,752 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting af99528764f945fdac0a6e11e7a1ce1f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733116700296 2024-12-02T05:18:23,752 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 82ca838021c04257b697b19f0d34e207, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733116701682 2024-12-02T05:18:23,752 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f6a5ea2267046bc92c7af2d1786920d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733116700357 2024-12-02T05:18:23,752 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4230e624f4a64e55bd0e2697feff5c35, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733116700494 2024-12-02T05:18:23,753 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e77057d7cf748e0bdc5e2137ff50b0f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733116701665 2024-12-02T05:18:23,767 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#B#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:23,767 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/e11060276ef045308fc4036ae00ede7d is 50, key is test_row_0/B:col10/1733116701684/Put/seqid=0 2024-12-02T05:18:23,773 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:23,776 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412027059aef121ea4aa4a44f89b391d252cb_ff61f31505788790cd78022602747873 store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:23,778 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412027059aef121ea4aa4a44f89b391d252cb_ff61f31505788790cd78022602747873, store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:23,779 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027059aef121ea4aa4a44f89b391d252cb_ff61f31505788790cd78022602747873 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:23,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742166_1342 (size=12139) 2024-12-02T05:18:23,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742167_1343 (size=4469) 2024-12-02T05:18:23,798 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#A#compaction#294 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:23,798 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/cbc49c7172c34b99a1755a8ca28a016a is 175, key is test_row_0/A:col10/1733116701684/Put/seqid=0 2024-12-02T05:18:23,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742168_1344 (size=31093) 2024-12-02T05:18:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:24,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T05:18:24,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:24,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:24,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:24,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:24,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:24,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:24,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ba5eb8cfd7d24ed096c0aa6e2b9896f2_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116702860/Put/seqid=0 2024-12-02T05:18:24,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742169_1345 (size=14594) 2024-12-02T05:18:24,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116764049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733116704032, max timestamp=1733116704059Num uniques 859; 2024-12-02T05:18:24,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733116704032, max timestamp=1733116704059Num uniques 859; 2024-12-02T05:18:24,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-02T05:18:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116764050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-02T05:18:24,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733116704032, max timestamp=1733116704059Num uniques 859; 2024-12-02T05:18:24,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-02T05:18:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116764093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116764094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116764100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116764167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116764167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,184 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/e11060276ef045308fc4036ae00ede7d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/e11060276ef045308fc4036ae00ede7d 2024-12-02T05:18:24,189 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ff61f31505788790cd78022602747873/B of ff61f31505788790cd78022602747873 into e11060276ef045308fc4036ae00ede7d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:24,189 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:24,189 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/B, priority=12, startTime=1733116703747; duration=0sec 2024-12-02T05:18:24,189 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:24,189 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:B 2024-12-02T05:18:24,189 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:18:24,190 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:18:24,190 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/C is initiating minor compaction (all files) 2024-12-02T05:18:24,191 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/C in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:24,191 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/c6e06f88fc01408380bd72c304e8ed66, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/8bb31c5dc8074a878feeed0a77a3162e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/22a7f32513a648109f3412281bd16da4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/82524a04db434915ab00ac6215f5a4f4] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=46.9 K 2024-12-02T05:18:24,191 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting c6e06f88fc01408380bd72c304e8ed66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733116700296 2024-12-02T05:18:24,191 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bb31c5dc8074a878feeed0a77a3162e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733116700357 2024-12-02T05:18:24,191 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 22a7f32513a648109f3412281bd16da4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733116700494 2024-12-02T05:18:24,192 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 82524a04db434915ab00ac6215f5a4f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733116701682 2024-12-02T05:18:24,201 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#C#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:24,201 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/f5ce0079e6a14238b7ec06ac2e64677c is 50, key is test_row_0/C:col10/1733116701684/Put/seqid=0 2024-12-02T05:18:24,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742170_1346 (size=12139) 2024-12-02T05:18:24,212 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/cbc49c7172c34b99a1755a8ca28a016a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/cbc49c7172c34b99a1755a8ca28a016a 2024-12-02T05:18:24,212 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/f5ce0079e6a14238b7ec06ac2e64677c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f5ce0079e6a14238b7ec06ac2e64677c 2024-12-02T05:18:24,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116764207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,219 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ff61f31505788790cd78022602747873/A of ff61f31505788790cd78022602747873 into cbc49c7172c34b99a1755a8ca28a016a(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:24,219 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:24,219 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/A, priority=12, startTime=1733116703747; duration=0sec 2024-12-02T05:18:24,219 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:24,219 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:A 2024-12-02T05:18:24,220 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ff61f31505788790cd78022602747873/C of ff61f31505788790cd78022602747873 into f5ce0079e6a14238b7ec06ac2e64677c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:24,220 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:24,220 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/C, priority=12, startTime=1733116703747; duration=0sec 2024-12-02T05:18:24,220 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:24,220 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:C 2024-12-02T05:18:24,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116764217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116764218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116764378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116764378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116764427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,433 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:24,438 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ba5eb8cfd7d24ed096c0aa6e2b9896f2_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ba5eb8cfd7d24ed096c0aa6e2b9896f2_ff61f31505788790cd78022602747873 2024-12-02T05:18:24,439 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/902143ff5b0645debe577a22cebe1586, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:24,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/902143ff5b0645debe577a22cebe1586 is 175, key is test_row_0/A:col10/1733116702860/Put/seqid=0 2024-12-02T05:18:24,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742171_1347 (size=39549) 2024-12-02T05:18:24,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116764440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116764440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T05:18:24,458 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:24,458 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:18:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-02T05:18:24,460 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:18:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T05:18:24,461 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:18:24,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:18:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T05:18:24,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T05:18:24,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:24,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:24,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:24,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:24,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:24,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116764698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116764698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116764737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116764758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116764758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:24,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T05:18:24,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:24,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:24,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:24,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:24,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T05:18:24,843 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/902143ff5b0645debe577a22cebe1586 2024-12-02T05:18:24,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/2ae6971edcef42b997981b63cd12c94a is 50, key is test_row_0/B:col10/1733116702860/Put/seqid=0 2024-12-02T05:18:24,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742172_1348 (size=12001) 2024-12-02T05:18:24,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/2ae6971edcef42b997981b63cd12c94a 2024-12-02T05:18:24,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/895c959bf5384bb6a84202f2a1a8b2cb is 50, key is test_row_0/C:col10/1733116702860/Put/seqid=0 2024-12-02T05:18:24,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742173_1349 (size=12001) 2024-12-02T05:18:24,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/895c959bf5384bb6a84202f2a1a8b2cb 2024-12-02T05:18:24,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/902143ff5b0645debe577a22cebe1586 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/902143ff5b0645debe577a22cebe1586 2024-12-02T05:18:24,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/902143ff5b0645debe577a22cebe1586, entries=200, sequenceid=108, filesize=38.6 K 2024-12-02T05:18:24,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/2ae6971edcef42b997981b63cd12c94a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/2ae6971edcef42b997981b63cd12c94a 2024-12-02T05:18:24,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T05:18:24,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:24,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:24,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:24,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:24,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:24,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/2ae6971edcef42b997981b63cd12c94a, entries=150, sequenceid=108, filesize=11.7 K 2024-12-02T05:18:24,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/895c959bf5384bb6a84202f2a1a8b2cb as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/895c959bf5384bb6a84202f2a1a8b2cb 2024-12-02T05:18:24,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/895c959bf5384bb6a84202f2a1a8b2cb, entries=150, sequenceid=108, filesize=11.7 K 2024-12-02T05:18:24,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for ff61f31505788790cd78022602747873 in 908ms, sequenceid=108, compaction requested=false 2024-12-02T05:18:24,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:25,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T05:18:25,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:25,073 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=134.18 KB heapSize=361.55 KB 2024-12-02T05:18:25,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:25,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:25,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:25,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:25,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:25,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:25,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c63200a738284769a7964122c388d4a4_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116704091/Put/seqid=0 2024-12-02T05:18:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T05:18:25,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742174_1350 (size=12304) 2024-12-02T05:18:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:25,105 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c63200a738284769a7964122c388d4a4_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c63200a738284769a7964122c388d4a4_ff61f31505788790cd78022602747873 2024-12-02T05:18:25,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/beefa5df71de42fa81d56f33a27e00e4, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:25,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/beefa5df71de42fa81d56f33a27e00e4 is 175, key is test_row_0/A:col10/1733116704091/Put/seqid=0 2024-12-02T05:18:25,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742175_1351 (size=31105) 2024-12-02T05:18:25,117 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/beefa5df71de42fa81d56f33a27e00e4 2024-12-02T05:18:25,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/2d6a07a310b54269bd2275e080ea6f16 is 50, key is test_row_0/B:col10/1733116704091/Put/seqid=0 2024-12-02T05:18:25,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742176_1352 (size=12151) 2024-12-02T05:18:25,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:25,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116765238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:25,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116765245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:25,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116765258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:25,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116765268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:25,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:25,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116765277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:25,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:25,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116765357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:25,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116765378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:25,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:25,531 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/2d6a07a310b54269bd2275e080ea6f16 2024-12-02T05:18:25,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/d4765abf1d4846fcb52a744b8d1879db is 50, key is test_row_0/C:col10/1733116704091/Put/seqid=0 2024-12-02T05:18:25,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742177_1353 (size=12151) 2024-12-02T05:18:25,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116765578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T05:18:25,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116765587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:25,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116765897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:25,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:25,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116765908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:25,970 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/d4765abf1d4846fcb52a744b8d1879db 2024-12-02T05:18:25,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/beefa5df71de42fa81d56f33a27e00e4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/beefa5df71de42fa81d56f33a27e00e4 2024-12-02T05:18:25,983 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/beefa5df71de42fa81d56f33a27e00e4, entries=150, sequenceid=143, filesize=30.4 K 2024-12-02T05:18:25,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/2d6a07a310b54269bd2275e080ea6f16 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/2d6a07a310b54269bd2275e080ea6f16 2024-12-02T05:18:25,995 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/2d6a07a310b54269bd2275e080ea6f16, entries=150, sequenceid=143, filesize=11.9 K 2024-12-02T05:18:25,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/d4765abf1d4846fcb52a744b8d1879db as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/d4765abf1d4846fcb52a744b8d1879db 2024-12-02T05:18:25,999 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/d4765abf1d4846fcb52a744b8d1879db, entries=150, sequenceid=143, filesize=11.9 K 2024-12-02T05:18:26,001 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~361.50 KB/370176, currentSize=53.67 KB/54960 for ff61f31505788790cd78022602747873 in 928ms, sequenceid=143, compaction requested=true 2024-12-02T05:18:26,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:26,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:26,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-02T05:18:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-02T05:18:26,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-02T05:18:26,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5410 sec 2024-12-02T05:18:26,005 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.5460 sec 2024-12-02T05:18:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:26,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T05:18:26,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:26,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:26,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:26,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:26,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:26,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:26,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026045f53963cf49f88a77d4fe918c7147_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116705243/Put/seqid=0 2024-12-02T05:18:26,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742178_1354 (size=14794) 2024-12-02T05:18:26,307 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:26,314 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026045f53963cf49f88a77d4fe918c7147_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026045f53963cf49f88a77d4fe918c7147_ff61f31505788790cd78022602747873 2024-12-02T05:18:26,315 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/901054e29c4e44669c94870237198664, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:26,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/901054e29c4e44669c94870237198664 is 175, key is test_row_0/A:col10/1733116705243/Put/seqid=0 2024-12-02T05:18:26,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742179_1355 (size=39749) 2024-12-02T05:18:26,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116766336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:26,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116766342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:26,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116766344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:26,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:26,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:26,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116766407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:26,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:26,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116766427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:26,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:26,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116766458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116766458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116766458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T05:18:26,607 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:26,607 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:18:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-02T05:18:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T05:18:26,609 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:18:26,610 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:18:26,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:18:26,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116766667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:26,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116766667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116766668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T05:18:26,733 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/901054e29c4e44669c94870237198664 2024-12-02T05:18:26,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/494f641a6356430995e6b82ecf2f4cff is 50, key is test_row_0/B:col10/1733116705243/Put/seqid=0 2024-12-02T05:18:26,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742180_1356 (size=12151) 2024-12-02T05:18:26,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T05:18:26,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:26,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:26,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:26,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:26,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:26,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T05:18:26,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:26,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:26,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:26,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:26,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T05:18:26,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116766978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:26,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:26,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116766978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116766978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:27,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T05:18:27,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:27,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:27,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:27,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:27,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:27,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/494f641a6356430995e6b82ecf2f4cff 2024-12-02T05:18:27,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/967bc780d921473d95f7ed6af31a2b97 is 50, key is test_row_0/C:col10/1733116705243/Put/seqid=0 2024-12-02T05:18:27,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742181_1357 (size=12151) 2024-12-02T05:18:27,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T05:18:27,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:27,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:27,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:27,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:27,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T05:18:27,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T05:18:27,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:27,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:27,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:27,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:27,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:27,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116767417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:27,424 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in ff61f31505788790cd78022602747873 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:18:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:27,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116767443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:27,448 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in ff61f31505788790cd78022602747873 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:18:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:27,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116767487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:27,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:27,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:27,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:27,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116767497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:27,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116767497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:27,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T05:18:27,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:27,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:27,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:27,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:27,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:27,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/967bc780d921473d95f7ed6af31a2b97 2024-12-02T05:18:27,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/901054e29c4e44669c94870237198664 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/901054e29c4e44669c94870237198664 2024-12-02T05:18:27,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/901054e29c4e44669c94870237198664, entries=200, sequenceid=155, filesize=38.8 K 2024-12-02T05:18:27,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/494f641a6356430995e6b82ecf2f4cff as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/494f641a6356430995e6b82ecf2f4cff 2024-12-02T05:18:27,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/494f641a6356430995e6b82ecf2f4cff, entries=150, sequenceid=155, filesize=11.9 K 2024-12-02T05:18:27,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/967bc780d921473d95f7ed6af31a2b97 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/967bc780d921473d95f7ed6af31a2b97 2024-12-02T05:18:27,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/967bc780d921473d95f7ed6af31a2b97, entries=150, sequenceid=155, filesize=11.9 K 2024-12-02T05:18:27,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ff61f31505788790cd78022602747873 in 1292ms, sequenceid=155, compaction requested=true 2024-12-02T05:18:27,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:27,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:18:27,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:27,574 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:18:27,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:18:27,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:27,574 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:18:27,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:18:27,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:27,575 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:18:27,575 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141496 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:18:27,575 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/A is initiating minor compaction (all files) 2024-12-02T05:18:27,575 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/B is initiating minor compaction (all files) 2024-12-02T05:18:27,575 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/A in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:27,575 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/B in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:27,575 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/e11060276ef045308fc4036ae00ede7d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/2ae6971edcef42b997981b63cd12c94a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/2d6a07a310b54269bd2275e080ea6f16, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/494f641a6356430995e6b82ecf2f4cff] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=47.3 K 2024-12-02T05:18:27,575 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/cbc49c7172c34b99a1755a8ca28a016a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/902143ff5b0645debe577a22cebe1586, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/beefa5df71de42fa81d56f33a27e00e4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/901054e29c4e44669c94870237198664] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=138.2 K 2024-12-02T05:18:27,575 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:27,575 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/cbc49c7172c34b99a1755a8ca28a016a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/902143ff5b0645debe577a22cebe1586, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/beefa5df71de42fa81d56f33a27e00e4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/901054e29c4e44669c94870237198664] 2024-12-02T05:18:27,575 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting e11060276ef045308fc4036ae00ede7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733116701682 2024-12-02T05:18:27,576 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ae6971edcef42b997981b63cd12c94a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733116702859 2024-12-02T05:18:27,576 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbc49c7172c34b99a1755a8ca28a016a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733116701682 2024-12-02T05:18:27,576 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 902143ff5b0645debe577a22cebe1586, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733116702859 2024-12-02T05:18:27,576 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d6a07a310b54269bd2275e080ea6f16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733116704081 2024-12-02T05:18:27,576 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 494f641a6356430995e6b82ecf2f4cff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733116705215 2024-12-02T05:18:27,576 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting beefa5df71de42fa81d56f33a27e00e4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733116704081 2024-12-02T05:18:27,577 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 901054e29c4e44669c94870237198664, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733116705215 2024-12-02T05:18:27,585 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:27,588 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202dbef16b3d89849f5b77d00df2817bd02_ff61f31505788790cd78022602747873 store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:27,588 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#B#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:27,589 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/671c49a9c2c349dea87913a546db880a is 50, key is test_row_0/B:col10/1733116705243/Put/seqid=0 2024-12-02T05:18:27,590 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202dbef16b3d89849f5b77d00df2817bd02_ff61f31505788790cd78022602747873, store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:27,591 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202dbef16b3d89849f5b77d00df2817bd02_ff61f31505788790cd78022602747873 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:27,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742182_1358 (size=12425) 2024-12-02T05:18:27,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742183_1359 (size=4469) 2024-12-02T05:18:27,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T05:18:27,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:27,680 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T05:18:27,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:27,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:27,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:27,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:27,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:27,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:27,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b69f621965a54fdfbaf9bfe399f36ab4_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116706324/Put/seqid=0 2024-12-02T05:18:27,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742184_1360 (size=12304) 2024-12-02T05:18:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T05:18:27,998 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/671c49a9c2c349dea87913a546db880a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/671c49a9c2c349dea87913a546db880a 2024-12-02T05:18:28,002 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ff61f31505788790cd78022602747873/B of ff61f31505788790cd78022602747873 into 671c49a9c2c349dea87913a546db880a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:28,002 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:28,002 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/B, priority=12, startTime=1733116707574; duration=0sec 2024-12-02T05:18:28,002 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:28,002 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:B 2024-12-02T05:18:28,002 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:18:28,003 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:18:28,003 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/C is initiating minor compaction (all files) 2024-12-02T05:18:28,004 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/C in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:28,004 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f5ce0079e6a14238b7ec06ac2e64677c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/895c959bf5384bb6a84202f2a1a8b2cb, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/d4765abf1d4846fcb52a744b8d1879db, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/967bc780d921473d95f7ed6af31a2b97] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=47.3 K 2024-12-02T05:18:28,004 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting f5ce0079e6a14238b7ec06ac2e64677c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733116701682 2024-12-02T05:18:28,004 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 895c959bf5384bb6a84202f2a1a8b2cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733116702859 2024-12-02T05:18:28,004 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting d4765abf1d4846fcb52a744b8d1879db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733116704081 2024-12-02T05:18:28,005 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 967bc780d921473d95f7ed6af31a2b97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733116705215 2024-12-02T05:18:28,007 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#A#compaction#305 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:28,007 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/5c9d7f6689c644c395cfa028237d5a95 is 175, key is test_row_0/A:col10/1733116705243/Put/seqid=0 2024-12-02T05:18:28,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742185_1361 (size=31379) 2024-12-02T05:18:28,014 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#C#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:28,015 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/5c9d7f6689c644c395cfa028237d5a95 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/5c9d7f6689c644c395cfa028237d5a95 2024-12-02T05:18:28,015 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/f7d49cac67f74ea99ce07b876c332ddf is 50, key is test_row_0/C:col10/1733116705243/Put/seqid=0 2024-12-02T05:18:28,034 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ff61f31505788790cd78022602747873/A of ff61f31505788790cd78022602747873 into 5c9d7f6689c644c395cfa028237d5a95(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:28,034 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:28,034 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/A, priority=12, startTime=1733116707574; duration=0sec 2024-12-02T05:18:28,034 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:28,034 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:A 2024-12-02T05:18:28,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742186_1362 (size=12425) 2024-12-02T05:18:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:28,105 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b69f621965a54fdfbaf9bfe399f36ab4_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b69f621965a54fdfbaf9bfe399f36ab4_ff61f31505788790cd78022602747873 2024-12-02T05:18:28,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/2f7f560ae71f48e2913bf8f6a3afb884, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:28,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/2f7f560ae71f48e2913bf8f6a3afb884 is 175, key is test_row_0/A:col10/1733116706324/Put/seqid=0 2024-12-02T05:18:28,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742187_1363 (size=31105) 2024-12-02T05:18:28,446 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/f7d49cac67f74ea99ce07b876c332ddf as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f7d49cac67f74ea99ce07b876c332ddf 2024-12-02T05:18:28,454 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ff61f31505788790cd78022602747873/C of ff61f31505788790cd78022602747873 into f7d49cac67f74ea99ce07b876c332ddf(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:28,454 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:28,454 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/C, priority=12, startTime=1733116707574; duration=0sec 2024-12-02T05:18:28,454 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:28,454 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:C 2024-12-02T05:18:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:28,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:28,511 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/2f7f560ae71f48e2913bf8f6a3afb884 2024-12-02T05:18:28,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/b11d8f2ea97648899c6364b23a5a50a5 is 50, key is test_row_0/B:col10/1733116706324/Put/seqid=0 2024-12-02T05:18:28,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742188_1364 (size=12151) 2024-12-02T05:18:28,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116768526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:28,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:28,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116768533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:28,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116768536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:28,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116768647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:28,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116768647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:28,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116768657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:28,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T05:18:28,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116768858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:28,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116768858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:28,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116768867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:28,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:28,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:28,935 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/b11d8f2ea97648899c6364b23a5a50a5 2024-12-02T05:18:28,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/94364cfd15e34d6583afb67fc364cab6 is 50, key is test_row_0/C:col10/1733116706324/Put/seqid=0 2024-12-02T05:18:28,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742189_1365 (size=12151) 2024-12-02T05:18:29,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116769167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:29,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116769167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:29,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116769178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:29,351 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/94364cfd15e34d6583afb67fc364cab6 2024-12-02T05:18:29,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/2f7f560ae71f48e2913bf8f6a3afb884 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2f7f560ae71f48e2913bf8f6a3afb884 2024-12-02T05:18:29,370 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2f7f560ae71f48e2913bf8f6a3afb884, entries=150, sequenceid=175, filesize=30.4 K 2024-12-02T05:18:29,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/b11d8f2ea97648899c6364b23a5a50a5 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/b11d8f2ea97648899c6364b23a5a50a5 2024-12-02T05:18:29,377 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/b11d8f2ea97648899c6364b23a5a50a5, entries=150, sequenceid=175, filesize=11.9 K 2024-12-02T05:18:29,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/94364cfd15e34d6583afb67fc364cab6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/94364cfd15e34d6583afb67fc364cab6 2024-12-02T05:18:29,385 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/94364cfd15e34d6583afb67fc364cab6, entries=150, sequenceid=175, filesize=11.9 K 2024-12-02T05:18:29,386 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ff61f31505788790cd78022602747873 in 1706ms, sequenceid=175, compaction requested=false 2024-12-02T05:18:29,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:29,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:29,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-02T05:18:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-02T05:18:29,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-02T05:18:29,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7780 sec 2024-12-02T05:18:29,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.7860 sec 2024-12-02T05:18:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:29,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T05:18:29,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:29,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:29,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:29,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:29,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:29,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:29,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202280bb4acbaa34dab850d9da0f4fd5af8_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116708524/Put/seqid=0 2024-12-02T05:18:29,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742190_1366 (size=14794) 2024-12-02T05:18:29,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116769502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:29,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116769504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:29,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116769618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116769618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:29,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116769677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:29,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116769688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:29,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116769688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:29,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116769827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116769827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:29,862 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:29,865 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202280bb4acbaa34dab850d9da0f4fd5af8_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202280bb4acbaa34dab850d9da0f4fd5af8_ff61f31505788790cd78022602747873 2024-12-02T05:18:29,866 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/c1d80d3019f5449abb57ad85a2d27fc2, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:29,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/c1d80d3019f5449abb57ad85a2d27fc2 is 175, key is test_row_0/A:col10/1733116708524/Put/seqid=0 2024-12-02T05:18:29,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742191_1367 (size=39749) 2024-12-02T05:18:30,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116770138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:30,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116770138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:30,270 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/c1d80d3019f5449abb57ad85a2d27fc2 2024-12-02T05:18:30,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/dd8bac6d66d9425da06dc8c8f726b543 is 50, key is test_row_0/B:col10/1733116708524/Put/seqid=0 2024-12-02T05:18:30,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742192_1368 (size=12151) 2024-12-02T05:18:30,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/dd8bac6d66d9425da06dc8c8f726b543 2024-12-02T05:18:30,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/1bf877fb6c2d41bdbb3badbaa364afba is 50, key is test_row_0/C:col10/1733116708524/Put/seqid=0 2024-12-02T05:18:30,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742193_1369 (size=12151) 2024-12-02T05:18:30,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116770647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:30,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116770657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:30,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116770698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:30,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116770699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:30,705 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in ff61f31505788790cd78022602747873 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:18:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:30,705 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in ff61f31505788790cd78022602747873 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:18:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:30,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:30,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/1bf877fb6c2d41bdbb3badbaa364afba 2024-12-02T05:18:30,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/c1d80d3019f5449abb57ad85a2d27fc2 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/c1d80d3019f5449abb57ad85a2d27fc2 2024-12-02T05:18:30,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/c1d80d3019f5449abb57ad85a2d27fc2, entries=200, sequenceid=191, filesize=38.8 K 2024-12-02T05:18:30,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/dd8bac6d66d9425da06dc8c8f726b543 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/dd8bac6d66d9425da06dc8c8f726b543 2024-12-02T05:18:30,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116770707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:30,714 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in ff61f31505788790cd78022602747873 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:18:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:30,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/dd8bac6d66d9425da06dc8c8f726b543, entries=150, sequenceid=191, filesize=11.9 K 2024-12-02T05:18:30,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/1bf877fb6c2d41bdbb3badbaa364afba as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/1bf877fb6c2d41bdbb3badbaa364afba 2024-12-02T05:18:30,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/1bf877fb6c2d41bdbb3badbaa364afba, entries=150, sequenceid=191, filesize=11.9 K 2024-12-02T05:18:30,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ff61f31505788790cd78022602747873 in 1268ms, sequenceid=191, compaction requested=true 2024-12-02T05:18:30,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:30,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:18:30,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:30,721 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:30,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:18:30,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:30,721 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:30,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:18:30,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:30,722 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:30,722 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:30,722 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/B is initiating minor compaction (all files) 2024-12-02T05:18:30,722 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/A is initiating minor compaction (all files) 2024-12-02T05:18:30,722 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/B in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:30,722 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/A in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:30,722 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/5c9d7f6689c644c395cfa028237d5a95, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2f7f560ae71f48e2913bf8f6a3afb884, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/c1d80d3019f5449abb57ad85a2d27fc2] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=99.8 K 2024-12-02T05:18:30,722 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/671c49a9c2c349dea87913a546db880a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/b11d8f2ea97648899c6364b23a5a50a5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/dd8bac6d66d9425da06dc8c8f726b543] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=35.9 K 2024-12-02T05:18:30,722 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:30,722 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/5c9d7f6689c644c395cfa028237d5a95, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2f7f560ae71f48e2913bf8f6a3afb884, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/c1d80d3019f5449abb57ad85a2d27fc2] 2024-12-02T05:18:30,722 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 671c49a9c2c349dea87913a546db880a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733116705215 2024-12-02T05:18:30,722 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c9d7f6689c644c395cfa028237d5a95, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733116705215 2024-12-02T05:18:30,723 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f7f560ae71f48e2913bf8f6a3afb884, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733116706324 2024-12-02T05:18:30,723 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting b11d8f2ea97648899c6364b23a5a50a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733116706324 2024-12-02T05:18:30,723 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1d80d3019f5449abb57ad85a2d27fc2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733116708510 2024-12-02T05:18:30,723 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting dd8bac6d66d9425da06dc8c8f726b543, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733116708510 2024-12-02T05:18:30,728 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:30,729 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#B#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:30,729 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/4295be8f15a54a8da9d0cc9ceb7f6d8e is 50, key is test_row_0/B:col10/1733116708524/Put/seqid=0 2024-12-02T05:18:30,733 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412029cddd56952e9478789b967bf7d0882e8_ff61f31505788790cd78022602747873 store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:30,734 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412029cddd56952e9478789b967bf7d0882e8_ff61f31505788790cd78022602747873, store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:30,734 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029cddd56952e9478789b967bf7d0882e8_ff61f31505788790cd78022602747873 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:30,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742194_1370 (size=12527) 2024-12-02T05:18:30,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742195_1371 (size=4469) 2024-12-02T05:18:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T05:18:30,778 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:30,778 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:18:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-02T05:18:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T05:18:30,780 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:18:30,781 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:18:30,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:18:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T05:18:30,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T05:18:30,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:30,933 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T05:18:30,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:30,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:30,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:30,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:30,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:30,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:30,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026b632919d6a4477f86c69bb8c2889a54_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116709500/Put/seqid=0 2024-12-02T05:18:30,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742196_1372 (size=12304) 2024-12-02T05:18:30,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:30,960 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026b632919d6a4477f86c69bb8c2889a54_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026b632919d6a4477f86c69bb8c2889a54_ff61f31505788790cd78022602747873 2024-12-02T05:18:30,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/929c50a29b074b509f05be15eb033523, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:30,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/929c50a29b074b509f05be15eb033523 is 175, key is test_row_0/A:col10/1733116709500/Put/seqid=0 2024-12-02T05:18:30,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742197_1373 (size=31105) 2024-12-02T05:18:30,983 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/929c50a29b074b509f05be15eb033523 2024-12-02T05:18:30,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/a0b371201b1644cbb900c168012c59a2 is 50, key is test_row_0/B:col10/1733116709500/Put/seqid=0 2024-12-02T05:18:31,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742198_1374 (size=12151) 2024-12-02T05:18:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T05:18:31,142 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/4295be8f15a54a8da9d0cc9ceb7f6d8e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/4295be8f15a54a8da9d0cc9ceb7f6d8e 2024-12-02T05:18:31,143 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#A#compaction#315 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:31,143 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/ba191247da464943ba744393befe75bc is 175, key is test_row_0/A:col10/1733116708524/Put/seqid=0 2024-12-02T05:18:31,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742199_1375 (size=31481) 2024-12-02T05:18:31,147 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff61f31505788790cd78022602747873/B of ff61f31505788790cd78022602747873 into 4295be8f15a54a8da9d0cc9ceb7f6d8e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:31,147 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:31,147 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/B, priority=13, startTime=1733116710721; duration=0sec 2024-12-02T05:18:31,147 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:31,147 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:B 2024-12-02T05:18:31,147 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:31,149 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:31,149 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/C is initiating minor compaction (all files) 2024-12-02T05:18:31,149 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/C in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:31,149 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f7d49cac67f74ea99ce07b876c332ddf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/94364cfd15e34d6583afb67fc364cab6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/1bf877fb6c2d41bdbb3badbaa364afba] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=35.9 K 2024-12-02T05:18:31,149 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting f7d49cac67f74ea99ce07b876c332ddf, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733116705215 2024-12-02T05:18:31,150 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 94364cfd15e34d6583afb67fc364cab6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733116706324 2024-12-02T05:18:31,150 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bf877fb6c2d41bdbb3badbaa364afba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733116708510 2024-12-02T05:18:31,151 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/ba191247da464943ba744393befe75bc as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/ba191247da464943ba744393befe75bc 2024-12-02T05:18:31,158 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#C#compaction#318 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:31,159 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/b1f0e823632b41168f203ab7ff5c8b35 is 50, key is test_row_0/C:col10/1733116708524/Put/seqid=0 2024-12-02T05:18:31,159 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff61f31505788790cd78022602747873/A of ff61f31505788790cd78022602747873 into ba191247da464943ba744393befe75bc(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:31,159 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:31,159 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/A, priority=13, startTime=1733116710721; duration=0sec 2024-12-02T05:18:31,160 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:31,160 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:A 2024-12-02T05:18:31,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742200_1376 (size=12527) 2024-12-02T05:18:31,167 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/b1f0e823632b41168f203ab7ff5c8b35 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/b1f0e823632b41168f203ab7ff5c8b35 2024-12-02T05:18:31,170 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff61f31505788790cd78022602747873/C of ff61f31505788790cd78022602747873 into b1f0e823632b41168f203ab7ff5c8b35(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:31,170 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:31,170 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/C, priority=13, startTime=1733116710721; duration=0sec 2024-12-02T05:18:31,170 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:31,170 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:C 2024-12-02T05:18:31,278 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T05:18:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T05:18:31,414 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/a0b371201b1644cbb900c168012c59a2 2024-12-02T05:18:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/f56993941f794bb6b00897d18d0775a2 is 50, key is test_row_0/C:col10/1733116709500/Put/seqid=0 2024-12-02T05:18:31,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742201_1377 (size=12151) 2024-12-02T05:18:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:31,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:31,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116771695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:31,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116771701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:31,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116771807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:31,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:31,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116771817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:31,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:31,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:31,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:31,830 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/f56993941f794bb6b00897d18d0775a2 2024-12-02T05:18:31,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/929c50a29b074b509f05be15eb033523 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/929c50a29b074b509f05be15eb033523 2024-12-02T05:18:31,843 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/929c50a29b074b509f05be15eb033523, entries=150, sequenceid=210, filesize=30.4 K 2024-12-02T05:18:31,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/a0b371201b1644cbb900c168012c59a2 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/a0b371201b1644cbb900c168012c59a2 2024-12-02T05:18:31,847 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/a0b371201b1644cbb900c168012c59a2, entries=150, sequenceid=210, filesize=11.9 K 2024-12-02T05:18:31,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/f56993941f794bb6b00897d18d0775a2 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f56993941f794bb6b00897d18d0775a2 2024-12-02T05:18:31,850 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f56993941f794bb6b00897d18d0775a2, entries=150, sequenceid=210, filesize=11.9 K 2024-12-02T05:18:31,851 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ff61f31505788790cd78022602747873 in 918ms, sequenceid=210, compaction requested=false 2024-12-02T05:18:31,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:31,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:31,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-02T05:18:31,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-02T05:18:31,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-02T05:18:31,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0730 sec 2024-12-02T05:18:31,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.0780 sec 2024-12-02T05:18:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T05:18:31,917 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:31,918 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:18:31,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-02T05:18:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T05:18:31,919 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:18:31,920 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:18:31,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:18:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:32,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T05:18:32,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:32,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:32,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:32,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:32,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:32,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T05:18:32,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120273ba079109f44008af5c52e80eb5825d_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116711694/Put/seqid=0 2024-12-02T05:18:32,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742202_1378 (size=14794) 2024-12-02T05:18:32,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T05:18:32,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:32,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:32,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:32,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116772077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:32,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116772077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:32,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116772187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:32,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:32,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116772187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:32,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:32,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T05:18:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:32,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T05:18:32,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T05:18:32,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:32,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:32,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:32,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116772397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116772397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:32,438 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:32,441 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120273ba079109f44008af5c52e80eb5825d_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120273ba079109f44008af5c52e80eb5825d_ff61f31505788790cd78022602747873 2024-12-02T05:18:32,442 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/5f7da2f802fe4611b6a368e127c8e647, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:32,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/5f7da2f802fe4611b6a368e127c8e647 is 175, key is test_row_0/A:col10/1733116711694/Put/seqid=0 2024-12-02T05:18:32,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742203_1379 (size=39749) 2024-12-02T05:18:32,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T05:18:32,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:32,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:32,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:32,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T05:18:32,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T05:18:32,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:32,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:32,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:32,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116772707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116772707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:32,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116772717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:32,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116772717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:32,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116772717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:32,722 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in ff61f31505788790cd78022602747873 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:18:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,723 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in ff61f31505788790cd78022602747873 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:18:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:32,724 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in ff61f31505788790cd78022602747873 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:18:32,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:32,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:32,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T05:18:32,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:32,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:32,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:32,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,846 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/5f7da2f802fe4611b6a368e127c8e647 2024-12-02T05:18:32,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/7218f10bbac944c58587883cc2925cd7 is 50, key is test_row_0/B:col10/1733116711694/Put/seqid=0 2024-12-02T05:18:32,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742204_1380 (size=12151) 2024-12-02T05:18:32,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T05:18:32,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:32,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:32,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:32,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T05:18:33,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T05:18:33,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:33,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:33,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:33,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:33,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:33,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:33,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:33,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116773217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:33,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:33,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116773217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:33,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:33,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:33,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:33,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/7218f10bbac944c58587883cc2925cd7 2024-12-02T05:18:33,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/156830278a204fc1b53cd548b6ee2183 is 50, key is test_row_0/C:col10/1733116711694/Put/seqid=0 2024-12-02T05:18:33,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742205_1381 (size=12151) 2024-12-02T05:18:33,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T05:18:33,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:33,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:33,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:33,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:33,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:33,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:33,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T05:18:33,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:33,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:33,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:33,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:33,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:33,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:33,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T05:18:33,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:33,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:33,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:33,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:33,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:33,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/156830278a204fc1b53cd548b6ee2183 2024-12-02T05:18:33,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/5f7da2f802fe4611b6a368e127c8e647 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/5f7da2f802fe4611b6a368e127c8e647 2024-12-02T05:18:33,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/5f7da2f802fe4611b6a368e127c8e647, entries=200, sequenceid=227, filesize=38.8 K 2024-12-02T05:18:33,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/7218f10bbac944c58587883cc2925cd7 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/7218f10bbac944c58587883cc2925cd7 2024-12-02T05:18:33,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/7218f10bbac944c58587883cc2925cd7, entries=150, sequenceid=227, filesize=11.9 K 2024-12-02T05:18:33,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/156830278a204fc1b53cd548b6ee2183 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/156830278a204fc1b53cd548b6ee2183 2024-12-02T05:18:33,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/156830278a204fc1b53cd548b6ee2183, entries=150, sequenceid=227, filesize=11.9 K 2024-12-02T05:18:33,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ff61f31505788790cd78022602747873 in 1660ms, sequenceid=227, compaction requested=true 2024-12-02T05:18:33,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:33,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:18:33,681 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:33,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:33,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:18:33,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:33,681 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:33,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:18:33,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:33,682 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:33,682 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/A is initiating minor compaction (all files) 2024-12-02T05:18:33,683 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:33,683 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/A in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:33,683 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/B is initiating minor compaction (all files) 2024-12-02T05:18:33,683 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/ba191247da464943ba744393befe75bc, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/929c50a29b074b509f05be15eb033523, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/5f7da2f802fe4611b6a368e127c8e647] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=99.9 K 2024-12-02T05:18:33,683 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/B in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:33,683 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:33,683 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/ba191247da464943ba744393befe75bc, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/929c50a29b074b509f05be15eb033523, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/5f7da2f802fe4611b6a368e127c8e647] 2024-12-02T05:18:33,683 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/4295be8f15a54a8da9d0cc9ceb7f6d8e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/a0b371201b1644cbb900c168012c59a2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/7218f10bbac944c58587883cc2925cd7] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=36.0 K 2024-12-02T05:18:33,683 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba191247da464943ba744393befe75bc, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733116708510 2024-12-02T05:18:33,683 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 4295be8f15a54a8da9d0cc9ceb7f6d8e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733116708510 2024-12-02T05:18:33,683 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 929c50a29b074b509f05be15eb033523, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733116709487 2024-12-02T05:18:33,683 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting a0b371201b1644cbb900c168012c59a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733116709487 2024-12-02T05:18:33,683 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f7da2f802fe4611b6a368e127c8e647, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733116711689 2024-12-02T05:18:33,683 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 7218f10bbac944c58587883cc2925cd7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733116711694 2024-12-02T05:18:33,688 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:33,689 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#B#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:33,690 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/daf3dd2efe0f4caa8f7c4aa2cfa996cb is 50, key is test_row_0/B:col10/1733116711694/Put/seqid=0 2024-12-02T05:18:33,690 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412025a4b7155a78147fd8f7354c6fc6fcacc_ff61f31505788790cd78022602747873 store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:33,692 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412025a4b7155a78147fd8f7354c6fc6fcacc_ff61f31505788790cd78022602747873, store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:33,692 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025a4b7155a78147fd8f7354c6fc6fcacc_ff61f31505788790cd78022602747873 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:33,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742206_1382 (size=12629) 2024-12-02T05:18:33,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742207_1383 (size=4469) 2024-12-02T05:18:33,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T05:18:33,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:33,763 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T05:18:33,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:33,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:33,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:33,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:33,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:33,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020868bdc8782a4ef9bfde85d4144a8b29_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116712075/Put/seqid=0 2024-12-02T05:18:33,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742208_1384 (size=12304) 2024-12-02T05:18:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T05:18:34,097 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#A#compaction#324 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:34,098 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/8d2b9c96caf74b30889b87705347532b is 175, key is test_row_0/A:col10/1733116711694/Put/seqid=0 2024-12-02T05:18:34,100 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/daf3dd2efe0f4caa8f7c4aa2cfa996cb as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/daf3dd2efe0f4caa8f7c4aa2cfa996cb 2024-12-02T05:18:34,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742209_1385 (size=31583) 2024-12-02T05:18:34,106 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/8d2b9c96caf74b30889b87705347532b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/8d2b9c96caf74b30889b87705347532b 2024-12-02T05:18:34,106 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff61f31505788790cd78022602747873/B of ff61f31505788790cd78022602747873 into daf3dd2efe0f4caa8f7c4aa2cfa996cb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:34,106 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:34,107 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/B, priority=13, startTime=1733116713681; duration=0sec 2024-12-02T05:18:34,107 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:34,107 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:B 2024-12-02T05:18:34,107 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:34,108 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:34,108 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/C is initiating minor compaction (all files) 2024-12-02T05:18:34,108 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/C in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:34,108 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/b1f0e823632b41168f203ab7ff5c8b35, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f56993941f794bb6b00897d18d0775a2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/156830278a204fc1b53cd548b6ee2183] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=36.0 K 2024-12-02T05:18:34,109 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f0e823632b41168f203ab7ff5c8b35, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733116708510 2024-12-02T05:18:34,109 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting f56993941f794bb6b00897d18d0775a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733116709487 2024-12-02T05:18:34,109 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 156830278a204fc1b53cd548b6ee2183, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733116711694 2024-12-02T05:18:34,111 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff61f31505788790cd78022602747873/A of ff61f31505788790cd78022602747873 into 8d2b9c96caf74b30889b87705347532b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:34,111 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:34,111 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/A, priority=13, startTime=1733116713681; duration=0sec 2024-12-02T05:18:34,111 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:34,111 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:A 2024-12-02T05:18:34,117 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#C#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:34,118 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/659b4892343747d6a1caaa79a8e5be00 is 50, key is test_row_0/C:col10/1733116711694/Put/seqid=0 2024-12-02T05:18:34,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742210_1386 (size=12629) 2024-12-02T05:18:34,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:34,199 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020868bdc8782a4ef9bfde85d4144a8b29_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020868bdc8782a4ef9bfde85d4144a8b29_ff61f31505788790cd78022602747873 2024-12-02T05:18:34,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/bf9c1bf79d594efe92db953195bd0f2a, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:34,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/bf9c1bf79d594efe92db953195bd0f2a is 175, key is test_row_0/A:col10/1733116712075/Put/seqid=0 2024-12-02T05:18:34,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742211_1387 (size=31105) 2024-12-02T05:18:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:34,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:34,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:34,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116774277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:34,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116774278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:34,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116774387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:34,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:34,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116774397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:34,539 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/659b4892343747d6a1caaa79a8e5be00 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/659b4892343747d6a1caaa79a8e5be00 2024-12-02T05:18:34,542 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff61f31505788790cd78022602747873/C of ff61f31505788790cd78022602747873 into 659b4892343747d6a1caaa79a8e5be00(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:34,543 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:34,543 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/C, priority=13, startTime=1733116713681; duration=0sec 2024-12-02T05:18:34,543 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:34,543 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:C 2024-12-02T05:18:34,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116774597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:34,605 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/bf9c1bf79d594efe92db953195bd0f2a 2024-12-02T05:18:34,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/43e9c22f72a745d3b60ef8fc3b84da9d is 50, key is test_row_0/B:col10/1733116712075/Put/seqid=0 2024-12-02T05:18:34,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116774607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:34,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742212_1388 (size=12151) 2024-12-02T05:18:34,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116774908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:34,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116774917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:35,016 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/43e9c22f72a745d3b60ef8fc3b84da9d 2024-12-02T05:18:35,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/697c53c1201e41faaaa3688f17419f8c is 50, key is test_row_0/C:col10/1733116712075/Put/seqid=0 2024-12-02T05:18:35,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742213_1389 (size=12151) 2024-12-02T05:18:35,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116775417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:35,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:35,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:35,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:35,425 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/697c53c1201e41faaaa3688f17419f8c 2024-12-02T05:18:35,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116775428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:35,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/bf9c1bf79d594efe92db953195bd0f2a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/bf9c1bf79d594efe92db953195bd0f2a 2024-12-02T05:18:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:35,442 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/bf9c1bf79d594efe92db953195bd0f2a, entries=150, sequenceid=245, filesize=30.4 K 2024-12-02T05:18:35,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/43e9c22f72a745d3b60ef8fc3b84da9d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/43e9c22f72a745d3b60ef8fc3b84da9d 2024-12-02T05:18:35,450 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/43e9c22f72a745d3b60ef8fc3b84da9d, entries=150, sequenceid=245, filesize=11.9 K 2024-12-02T05:18:35,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/697c53c1201e41faaaa3688f17419f8c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/697c53c1201e41faaaa3688f17419f8c 2024-12-02T05:18:35,455 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/697c53c1201e41faaaa3688f17419f8c, entries=150, sequenceid=245, filesize=11.9 K 2024-12-02T05:18:35,455 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ff61f31505788790cd78022602747873 in 1693ms, sequenceid=245, compaction requested=false 2024-12-02T05:18:35,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:35,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:35,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-02T05:18:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-02T05:18:35,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-02T05:18:35,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5360 sec 2024-12-02T05:18:35,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.5400 sec 2024-12-02T05:18:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T05:18:36,088 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:36,089 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:18:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-02T05:18:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T05:18:36,091 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:18:36,092 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:18:36,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:18:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T05:18:36,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T05:18:36,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:36,245 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T05:18:36,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:36,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:36,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:36,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:36,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:36,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:36,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d4af9eee4b354252b8521eb787046f7e_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116714275/Put/seqid=0 2024-12-02T05:18:36,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742214_1390 (size=12454) 2024-12-02T05:18:36,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:36,320 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d4af9eee4b354252b8521eb787046f7e_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d4af9eee4b354252b8521eb787046f7e_ff61f31505788790cd78022602747873 2024-12-02T05:18:36,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/7e0411af53a7487b8ba23581ab6a6f53, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:36,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/7e0411af53a7487b8ba23581ab6a6f53 is 175, key is test_row_0/A:col10/1733116714275/Put/seqid=0 2024-12-02T05:18:36,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742215_1391 (size=31255) 2024-12-02T05:18:36,369 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/7e0411af53a7487b8ba23581ab6a6f53 2024-12-02T05:18:36,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/87622af0a32c4cac80d5420c520a9bd1 is 50, key is test_row_0/B:col10/1733116714275/Put/seqid=0 2024-12-02T05:18:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T05:18:36,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742216_1392 (size=12301) 2024-12-02T05:18:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:36,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:36,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116776549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:36,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116776560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:36,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:36,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116776667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:36,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116776677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T05:18:36,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55134 deadline: 1733116776747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:36,756 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in ff61f31505788790cd78022602747873 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:18:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:36,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55076 deadline: 1733116776757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:36,761 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_1' in ff61f31505788790cd78022602747873 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:18:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:36,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1733116776768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:36,772 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in ff61f31505788790cd78022602747873 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:18:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:36,849 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/87622af0a32c4cac80d5420c520a9bd1 2024-12-02T05:18:36,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/b3e7af1fc0b04394b1e477e8d77160ba is 50, key is test_row_0/C:col10/1733116714275/Put/seqid=0 2024-12-02T05:18:36,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742217_1393 (size=12301) 2024-12-02T05:18:36,881 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/b3e7af1fc0b04394b1e477e8d77160ba 2024-12-02T05:18:36,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/7e0411af53a7487b8ba23581ab6a6f53 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/7e0411af53a7487b8ba23581ab6a6f53 2024-12-02T05:18:36,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116776887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:36,902 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/7e0411af53a7487b8ba23581ab6a6f53, entries=150, sequenceid=262, filesize=30.5 K 2024-12-02T05:18:36,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/87622af0a32c4cac80d5420c520a9bd1 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/87622af0a32c4cac80d5420c520a9bd1 2024-12-02T05:18:36,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:36,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116776897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:36,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:36,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:36,916 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/87622af0a32c4cac80d5420c520a9bd1, entries=150, sequenceid=262, filesize=12.0 K 2024-12-02T05:18:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/b3e7af1fc0b04394b1e477e8d77160ba as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/b3e7af1fc0b04394b1e477e8d77160ba 2024-12-02T05:18:36,926 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/b3e7af1fc0b04394b1e477e8d77160ba, entries=150, sequenceid=262, filesize=12.0 K 2024-12-02T05:18:36,927 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ff61f31505788790cd78022602747873 in 682ms, sequenceid=262, compaction requested=true 2024-12-02T05:18:36,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:36,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:36,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-02T05:18:36,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-02T05:18:36,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-02T05:18:36,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 836 msec 2024-12-02T05:18:36,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 842 msec 2024-12-02T05:18:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:37,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T05:18:37,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:37,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:37,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:37,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:37,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:37,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T05:18:37,228 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:37,229 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:18:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-02T05:18:37,231 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:18:37,232 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:18:37,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:18:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T05:18:37,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202edca0c401a48456fa9aceac2491079f7_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116717215/Put/seqid=0 2024-12-02T05:18:37,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116777252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:37,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116777253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:37,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742218_1394 (size=14994) 2024-12-02T05:18:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T05:18:37,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116777357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:37,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116777368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:37,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-02T05:18:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:37,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:37,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-02T05:18:37,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:37,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:37,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:37,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:37,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T05:18:37,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116777570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:37,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116777579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:37,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:37,680 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:37,693 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202edca0c401a48456fa9aceac2491079f7_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202edca0c401a48456fa9aceac2491079f7_ff61f31505788790cd78022602747873 2024-12-02T05:18:37,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-02T05:18:37,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:37,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:37,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:37,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:37,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:37,700 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/14536438c29043be85135191a73eb259, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:37,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/14536438c29043be85135191a73eb259 is 175, key is test_row_0/A:col10/1733116717215/Put/seqid=0 2024-12-02T05:18:37,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742219_1395 (size=39949) 2024-12-02T05:18:37,724 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/14536438c29043be85135191a73eb259 2024-12-02T05:18:37,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/9b63b109c8e84994800da5a71285ae74 is 50, key is test_row_0/B:col10/1733116717215/Put/seqid=0 2024-12-02T05:18:37,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742220_1396 (size=12301) 2024-12-02T05:18:37,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/9b63b109c8e84994800da5a71285ae74 2024-12-02T05:18:37,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/faecfb86a0b34ee18d1e751936511cb2 is 50, key is test_row_0/C:col10/1733116717215/Put/seqid=0 2024-12-02T05:18:37,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-02T05:18:37,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:37,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:37,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:37,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:37,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T05:18:37,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742221_1397 (size=12301) 2024-12-02T05:18:37,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/faecfb86a0b34ee18d1e751936511cb2 2024-12-02T05:18:37,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/14536438c29043be85135191a73eb259 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/14536438c29043be85135191a73eb259 2024-12-02T05:18:37,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116777888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:37,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/14536438c29043be85135191a73eb259, entries=200, sequenceid=281, filesize=39.0 K 2024-12-02T05:18:37,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116777899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:37,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:37,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:37,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/9b63b109c8e84994800da5a71285ae74 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/9b63b109c8e84994800da5a71285ae74 2024-12-02T05:18:37,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/9b63b109c8e84994800da5a71285ae74, entries=150, sequenceid=281, filesize=12.0 K 2024-12-02T05:18:37,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/faecfb86a0b34ee18d1e751936511cb2 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/faecfb86a0b34ee18d1e751936511cb2 2024-12-02T05:18:37,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/faecfb86a0b34ee18d1e751936511cb2, entries=150, sequenceid=281, filesize=12.0 K 2024-12-02T05:18:37,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ff61f31505788790cd78022602747873 in 713ms, sequenceid=281, compaction requested=true 2024-12-02T05:18:37,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:37,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:18:37,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:37,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:18:37,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:37,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:18:37,931 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:18:37,931 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:18:37,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:37,935 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:18:37,936 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133892 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:18:37,936 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/A is initiating minor compaction (all files) 2024-12-02T05:18:37,936 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/B is initiating minor compaction (all files) 2024-12-02T05:18:37,937 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/A in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:37,937 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/8d2b9c96caf74b30889b87705347532b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/bf9c1bf79d594efe92db953195bd0f2a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/7e0411af53a7487b8ba23581ab6a6f53, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/14536438c29043be85135191a73eb259] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=130.8 K 2024-12-02T05:18:37,937 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:37,938 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/8d2b9c96caf74b30889b87705347532b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/bf9c1bf79d594efe92db953195bd0f2a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/7e0411af53a7487b8ba23581ab6a6f53, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/14536438c29043be85135191a73eb259] 2024-12-02T05:18:37,939 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d2b9c96caf74b30889b87705347532b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733116711694 2024-12-02T05:18:37,939 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf9c1bf79d594efe92db953195bd0f2a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733116712040 2024-12-02T05:18:37,939 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e0411af53a7487b8ba23581ab6a6f53, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733116714262 2024-12-02T05:18:37,940 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/B in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:37,940 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/daf3dd2efe0f4caa8f7c4aa2cfa996cb, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/43e9c22f72a745d3b60ef8fc3b84da9d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/87622af0a32c4cac80d5420c520a9bd1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/9b63b109c8e84994800da5a71285ae74] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=48.2 K 2024-12-02T05:18:37,941 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting daf3dd2efe0f4caa8f7c4aa2cfa996cb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733116711694 2024-12-02T05:18:37,942 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14536438c29043be85135191a73eb259, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733116716537 2024-12-02T05:18:37,942 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 43e9c22f72a745d3b60ef8fc3b84da9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733116712040 2024-12-02T05:18:37,943 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 87622af0a32c4cac80d5420c520a9bd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733116714262 2024-12-02T05:18:37,944 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b63b109c8e84994800da5a71285ae74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733116716540 2024-12-02T05:18:37,969 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:37,982 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:37,983 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/6de396be7dd3481683112f7308980310 is 50, key is test_row_0/B:col10/1733116717215/Put/seqid=0 2024-12-02T05:18:37,989 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202138b9129ea9647a3b1ee3c5bd8b37305_ff61f31505788790cd78022602747873 store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:37,992 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202138b9129ea9647a3b1ee3c5bd8b37305_ff61f31505788790cd78022602747873, store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:37,992 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202138b9129ea9647a3b1ee3c5bd8b37305_ff61f31505788790cd78022602747873 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:38,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-02T05:18:38,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:38,014 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T05:18:38,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:38,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:38,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:38,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:38,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:38,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:38,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742222_1398 (size=12915) 2024-12-02T05:18:38,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742223_1399 (size=4469) 2024-12-02T05:18:38,065 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#A#compaction#335 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:38,066 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/0289e486e10e417bac964c001dc66073 is 175, key is test_row_0/A:col10/1733116717215/Put/seqid=0 2024-12-02T05:18:38,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029ae0656e17f24d5684f32806a1fc5ee7_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116717250/Put/seqid=0 2024-12-02T05:18:38,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742224_1400 (size=31869) 2024-12-02T05:18:38,138 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/0289e486e10e417bac964c001dc66073 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/0289e486e10e417bac964c001dc66073 2024-12-02T05:18:38,146 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ff61f31505788790cd78022602747873/A of ff61f31505788790cd78022602747873 into 0289e486e10e417bac964c001dc66073(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:38,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742225_1401 (size=12454) 2024-12-02T05:18:38,146 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:38,146 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/A, priority=12, startTime=1733116717931; duration=0sec 2024-12-02T05:18:38,146 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:38,146 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:A 2024-12-02T05:18:38,147 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:18:38,149 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:18:38,149 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/C is initiating minor compaction (all files) 2024-12-02T05:18:38,149 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/C in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:38,150 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/659b4892343747d6a1caaa79a8e5be00, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/697c53c1201e41faaaa3688f17419f8c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/b3e7af1fc0b04394b1e477e8d77160ba, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/faecfb86a0b34ee18d1e751936511cb2] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=48.2 K 2024-12-02T05:18:38,150 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 659b4892343747d6a1caaa79a8e5be00, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733116711694 2024-12-02T05:18:38,152 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 697c53c1201e41faaaa3688f17419f8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733116712040 2024-12-02T05:18:38,153 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3e7af1fc0b04394b1e477e8d77160ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733116714262 2024-12-02T05:18:38,154 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting faecfb86a0b34ee18d1e751936511cb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733116716540 2024-12-02T05:18:38,185 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#C#compaction#338 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:38,185 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/830d210758e84d75830b610b40ca697a is 50, key is test_row_0/C:col10/1733116717215/Put/seqid=0 2024-12-02T05:18:38,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742226_1402 (size=12915) 2024-12-02T05:18:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T05:18:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:38,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. as already flushing 2024-12-02T05:18:38,438 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/6de396be7dd3481683112f7308980310 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/6de396be7dd3481683112f7308980310 2024-12-02T05:18:38,446 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ff61f31505788790cd78022602747873/B of ff61f31505788790cd78022602747873 into 6de396be7dd3481683112f7308980310(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:38,446 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:38,446 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/B, priority=12, startTime=1733116717931; duration=0sec 2024-12-02T05:18:38,447 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:38,447 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:B 2024-12-02T05:18:38,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:38,552 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029ae0656e17f24d5684f32806a1fc5ee7_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029ae0656e17f24d5684f32806a1fc5ee7_ff61f31505788790cd78022602747873 2024-12-02T05:18:38,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/62c702fd898b4e8abbc8927e7d524bfa, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:38,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/62c702fd898b4e8abbc8927e7d524bfa is 175, key is test_row_0/A:col10/1733116717250/Put/seqid=0 2024-12-02T05:18:38,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116778567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:38,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:38,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:38,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742227_1403 (size=31255) 2024-12-02T05:18:38,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116778575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:38,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:38,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:38,634 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/830d210758e84d75830b610b40ca697a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/830d210758e84d75830b610b40ca697a 2024-12-02T05:18:38,642 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ff61f31505788790cd78022602747873/C of ff61f31505788790cd78022602747873 into 830d210758e84d75830b610b40ca697a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:38,642 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:38,642 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/C, priority=12, startTime=1733116717931; duration=0sec 2024-12-02T05:18:38,643 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:38,643 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:C 2024-12-02T05:18:38,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116778687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:38,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:38,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116778698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:38,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:38,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:38,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:38,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116778907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:38,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:38,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116778917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:38,987 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/62c702fd898b4e8abbc8927e7d524bfa 2024-12-02T05:18:39,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/76f462efbab641eca262e97981e026f1 is 50, key is test_row_0/B:col10/1733116717250/Put/seqid=0 2024-12-02T05:18:39,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742228_1404 (size=12301) 2024-12-02T05:18:39,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116779217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:39,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116779227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:39,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:39,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:39,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T05:18:39,424 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/76f462efbab641eca262e97981e026f1 2024-12-02T05:18:39,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/a474e964b3b44819a57a04e4be965858 is 50, key is test_row_0/C:col10/1733116717250/Put/seqid=0 2024-12-02T05:18:39,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742229_1405 (size=12301) 2024-12-02T05:18:39,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55120 deadline: 1733116779727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:39,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:39,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:39,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:39,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55150 deadline: 1733116779737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:39,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:39,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:39,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:18:39,835 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/a474e964b3b44819a57a04e4be965858 2024-12-02T05:18:39,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/62c702fd898b4e8abbc8927e7d524bfa as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/62c702fd898b4e8abbc8927e7d524bfa 2024-12-02T05:18:39,844 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/62c702fd898b4e8abbc8927e7d524bfa, entries=150, sequenceid=294, filesize=30.5 K 2024-12-02T05:18:39,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/76f462efbab641eca262e97981e026f1 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/76f462efbab641eca262e97981e026f1 2024-12-02T05:18:39,848 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/76f462efbab641eca262e97981e026f1, entries=150, sequenceid=294, filesize=12.0 K 2024-12-02T05:18:39,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/a474e964b3b44819a57a04e4be965858 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/a474e964b3b44819a57a04e4be965858 2024-12-02T05:18:39,851 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/a474e964b3b44819a57a04e4be965858, entries=150, sequenceid=294, filesize=12.0 K 2024-12-02T05:18:39,852 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for ff61f31505788790cd78022602747873 in 1838ms, sequenceid=294, compaction requested=false 2024-12-02T05:18:39,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:39,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:39,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-02T05:18:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-02T05:18:39,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-02T05:18:39,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6210 sec 2024-12-02T05:18:39,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.6260 sec 2024-12-02T05:18:39,909 WARN [regionserver/f49461030c22:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-12-02T05:18:39,909 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/af99528764f945fdac0a6e11e7a1ce1f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2f6a5ea2267046bc92c7af2d1786920d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/4230e624f4a64e55bd0e2697feff5c35, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2e77057d7cf748e0bdc5e2137ff50b0f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/cbc49c7172c34b99a1755a8ca28a016a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/902143ff5b0645debe577a22cebe1586, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/beefa5df71de42fa81d56f33a27e00e4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/901054e29c4e44669c94870237198664, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/5c9d7f6689c644c395cfa028237d5a95, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2f7f560ae71f48e2913bf8f6a3afb884, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/c1d80d3019f5449abb57ad85a2d27fc2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/ba191247da464943ba744393befe75bc, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/929c50a29b074b509f05be15eb033523, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/5f7da2f802fe4611b6a368e127c8e647, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/8d2b9c96caf74b30889b87705347532b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/bf9c1bf79d594efe92db953195bd0f2a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/7e0411af53a7487b8ba23581ab6a6f53, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/14536438c29043be85135191a73eb259] to archive 2024-12-02T05:18:39,910 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:18:39,913 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/af99528764f945fdac0a6e11e7a1ce1f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/af99528764f945fdac0a6e11e7a1ce1f 2024-12-02T05:18:39,917 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2f6a5ea2267046bc92c7af2d1786920d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2f6a5ea2267046bc92c7af2d1786920d 2024-12-02T05:18:39,920 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/4230e624f4a64e55bd0e2697feff5c35 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/4230e624f4a64e55bd0e2697feff5c35 2024-12-02T05:18:39,927 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2e77057d7cf748e0bdc5e2137ff50b0f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2e77057d7cf748e0bdc5e2137ff50b0f 2024-12-02T05:18:39,931 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/cbc49c7172c34b99a1755a8ca28a016a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/cbc49c7172c34b99a1755a8ca28a016a 2024-12-02T05:18:39,933 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/902143ff5b0645debe577a22cebe1586 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/902143ff5b0645debe577a22cebe1586 2024-12-02T05:18:39,934 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/beefa5df71de42fa81d56f33a27e00e4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/beefa5df71de42fa81d56f33a27e00e4 2024-12-02T05:18:39,935 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/901054e29c4e44669c94870237198664 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/901054e29c4e44669c94870237198664 2024-12-02T05:18:39,938 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/5c9d7f6689c644c395cfa028237d5a95 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/5c9d7f6689c644c395cfa028237d5a95 2024-12-02T05:18:39,939 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2f7f560ae71f48e2913bf8f6a3afb884 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/2f7f560ae71f48e2913bf8f6a3afb884 2024-12-02T05:18:39,940 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/c1d80d3019f5449abb57ad85a2d27fc2 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/c1d80d3019f5449abb57ad85a2d27fc2 2024-12-02T05:18:39,942 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/ba191247da464943ba744393befe75bc to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/ba191247da464943ba744393befe75bc 2024-12-02T05:18:39,943 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/929c50a29b074b509f05be15eb033523 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/929c50a29b074b509f05be15eb033523 2024-12-02T05:18:39,945 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/5f7da2f802fe4611b6a368e127c8e647 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/5f7da2f802fe4611b6a368e127c8e647 2024-12-02T05:18:39,946 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/8d2b9c96caf74b30889b87705347532b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/8d2b9c96caf74b30889b87705347532b 2024-12-02T05:18:39,947 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/bf9c1bf79d594efe92db953195bd0f2a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/bf9c1bf79d594efe92db953195bd0f2a 2024-12-02T05:18:39,950 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/7e0411af53a7487b8ba23581ab6a6f53 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/7e0411af53a7487b8ba23581ab6a6f53 2024-12-02T05:18:39,951 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/14536438c29043be85135191a73eb259 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/14536438c29043be85135191a73eb259 2024-12-02T05:18:39,953 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/fb6d0e52f41b48fb8035573ea6ded859, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/0895b10665df4a7686c2068c4eb519d9, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/985151927d674c90a95e1cb6ee06aa58, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/e11060276ef045308fc4036ae00ede7d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/82ca838021c04257b697b19f0d34e207, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/2ae6971edcef42b997981b63cd12c94a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/2d6a07a310b54269bd2275e080ea6f16, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/671c49a9c2c349dea87913a546db880a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/494f641a6356430995e6b82ecf2f4cff, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/b11d8f2ea97648899c6364b23a5a50a5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/4295be8f15a54a8da9d0cc9ceb7f6d8e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/dd8bac6d66d9425da06dc8c8f726b543, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/a0b371201b1644cbb900c168012c59a2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/daf3dd2efe0f4caa8f7c4aa2cfa996cb, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/7218f10bbac944c58587883cc2925cd7, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/43e9c22f72a745d3b60ef8fc3b84da9d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/87622af0a32c4cac80d5420c520a9bd1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/9b63b109c8e84994800da5a71285ae74] to archive 2024-12-02T05:18:39,955 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:18:39,963 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/fb6d0e52f41b48fb8035573ea6ded859 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/fb6d0e52f41b48fb8035573ea6ded859 2024-12-02T05:18:39,964 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/0895b10665df4a7686c2068c4eb519d9 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/0895b10665df4a7686c2068c4eb519d9 2024-12-02T05:18:39,966 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/985151927d674c90a95e1cb6ee06aa58 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/985151927d674c90a95e1cb6ee06aa58 2024-12-02T05:18:39,967 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/e11060276ef045308fc4036ae00ede7d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/e11060276ef045308fc4036ae00ede7d 2024-12-02T05:18:39,971 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/82ca838021c04257b697b19f0d34e207 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/82ca838021c04257b697b19f0d34e207 2024-12-02T05:18:39,973 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/2ae6971edcef42b997981b63cd12c94a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/2ae6971edcef42b997981b63cd12c94a 2024-12-02T05:18:39,974 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/2d6a07a310b54269bd2275e080ea6f16 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/2d6a07a310b54269bd2275e080ea6f16 2024-12-02T05:18:39,975 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/671c49a9c2c349dea87913a546db880a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/671c49a9c2c349dea87913a546db880a 2024-12-02T05:18:39,976 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/494f641a6356430995e6b82ecf2f4cff to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/494f641a6356430995e6b82ecf2f4cff 2024-12-02T05:18:39,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/b11d8f2ea97648899c6364b23a5a50a5 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/b11d8f2ea97648899c6364b23a5a50a5 2024-12-02T05:18:39,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/4295be8f15a54a8da9d0cc9ceb7f6d8e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/4295be8f15a54a8da9d0cc9ceb7f6d8e 2024-12-02T05:18:39,980 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/dd8bac6d66d9425da06dc8c8f726b543 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/dd8bac6d66d9425da06dc8c8f726b543 2024-12-02T05:18:39,981 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/a0b371201b1644cbb900c168012c59a2 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/a0b371201b1644cbb900c168012c59a2 2024-12-02T05:18:39,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/daf3dd2efe0f4caa8f7c4aa2cfa996cb to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/daf3dd2efe0f4caa8f7c4aa2cfa996cb 2024-12-02T05:18:39,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/7218f10bbac944c58587883cc2925cd7 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/7218f10bbac944c58587883cc2925cd7 2024-12-02T05:18:39,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/43e9c22f72a745d3b60ef8fc3b84da9d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/43e9c22f72a745d3b60ef8fc3b84da9d 2024-12-02T05:18:39,987 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/87622af0a32c4cac80d5420c520a9bd1 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/87622af0a32c4cac80d5420c520a9bd1 2024-12-02T05:18:39,988 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/9b63b109c8e84994800da5a71285ae74 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/9b63b109c8e84994800da5a71285ae74 2024-12-02T05:18:39,990 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/c6e06f88fc01408380bd72c304e8ed66, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/8bb31c5dc8074a878feeed0a77a3162e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/22a7f32513a648109f3412281bd16da4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f5ce0079e6a14238b7ec06ac2e64677c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/82524a04db434915ab00ac6215f5a4f4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/895c959bf5384bb6a84202f2a1a8b2cb, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/d4765abf1d4846fcb52a744b8d1879db, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f7d49cac67f74ea99ce07b876c332ddf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/967bc780d921473d95f7ed6af31a2b97, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/94364cfd15e34d6583afb67fc364cab6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/b1f0e823632b41168f203ab7ff5c8b35, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/1bf877fb6c2d41bdbb3badbaa364afba, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f56993941f794bb6b00897d18d0775a2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/659b4892343747d6a1caaa79a8e5be00, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/156830278a204fc1b53cd548b6ee2183, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/697c53c1201e41faaaa3688f17419f8c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/b3e7af1fc0b04394b1e477e8d77160ba, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/faecfb86a0b34ee18d1e751936511cb2] to archive 2024-12-02T05:18:39,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:18:39,995 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/c6e06f88fc01408380bd72c304e8ed66 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/c6e06f88fc01408380bd72c304e8ed66 2024-12-02T05:18:39,996 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/8bb31c5dc8074a878feeed0a77a3162e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/8bb31c5dc8074a878feeed0a77a3162e 2024-12-02T05:18:39,997 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/22a7f32513a648109f3412281bd16da4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/22a7f32513a648109f3412281bd16da4 2024-12-02T05:18:39,998 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f5ce0079e6a14238b7ec06ac2e64677c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f5ce0079e6a14238b7ec06ac2e64677c 2024-12-02T05:18:39,999 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/82524a04db434915ab00ac6215f5a4f4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/82524a04db434915ab00ac6215f5a4f4 2024-12-02T05:18:40,000 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/895c959bf5384bb6a84202f2a1a8b2cb to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/895c959bf5384bb6a84202f2a1a8b2cb 2024-12-02T05:18:40,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/d4765abf1d4846fcb52a744b8d1879db to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/d4765abf1d4846fcb52a744b8d1879db 2024-12-02T05:18:40,002 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f7d49cac67f74ea99ce07b876c332ddf to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f7d49cac67f74ea99ce07b876c332ddf 2024-12-02T05:18:40,012 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/967bc780d921473d95f7ed6af31a2b97 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/967bc780d921473d95f7ed6af31a2b97 2024-12-02T05:18:40,016 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/94364cfd15e34d6583afb67fc364cab6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/94364cfd15e34d6583afb67fc364cab6 2024-12-02T05:18:40,019 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/b1f0e823632b41168f203ab7ff5c8b35 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/b1f0e823632b41168f203ab7ff5c8b35 2024-12-02T05:18:40,021 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/1bf877fb6c2d41bdbb3badbaa364afba to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/1bf877fb6c2d41bdbb3badbaa364afba 2024-12-02T05:18:40,022 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f56993941f794bb6b00897d18d0775a2 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f56993941f794bb6b00897d18d0775a2 2024-12-02T05:18:40,026 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/659b4892343747d6a1caaa79a8e5be00 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/659b4892343747d6a1caaa79a8e5be00 2024-12-02T05:18:40,028 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/156830278a204fc1b53cd548b6ee2183 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/156830278a204fc1b53cd548b6ee2183 2024-12-02T05:18:40,031 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/697c53c1201e41faaaa3688f17419f8c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/697c53c1201e41faaaa3688f17419f8c 2024-12-02T05:18:40,034 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/b3e7af1fc0b04394b1e477e8d77160ba to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/b3e7af1fc0b04394b1e477e8d77160ba 2024-12-02T05:18:40,035 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f49461030c22:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/faecfb86a0b34ee18d1e751936511cb2 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/faecfb86a0b34ee18d1e751936511cb2 2024-12-02T05:18:40,314 INFO [Thread-1509 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1509. 2024-12-02T05:18:40,314 DEBUG [Thread-1509 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:40,314 DEBUG [Thread-1509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:40,314 DEBUG [Thread-1509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:40,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:40,316 INFO [Thread-1505 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1505. 2024-12-02T05:18:40,316 DEBUG [Thread-1505 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:40,316 DEBUG [Thread-1505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:40,316 DEBUG [Thread-1505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:40,317 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:40,318 INFO [Thread-1513 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1513. 2024-12-02T05:18:40,318 DEBUG [Thread-1513 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:40,318 DEBUG [Thread-1513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:40,318 DEBUG [Thread-1513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:40,318 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:40,318 INFO [Thread-1507 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1507. 2024-12-02T05:18:40,318 DEBUG [Thread-1507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:40,318 DEBUG [Thread-1507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:40,318 DEBUG [Thread-1507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:40,319 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:40,319 INFO [Thread-1511 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1511. 2024-12-02T05:18:40,319 DEBUG [Thread-1511 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:40,319 DEBUG [Thread-1511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:40,319 DEBUG [Thread-1511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:40,319 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on ff61f31505788790cd78022602747873 2024-12-02T05:18:40,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T05:18:40,751 INFO [Thread-1498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1498. 2024-12-02T05:18:40,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:40,751 DEBUG [Thread-1498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:40,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:40,751 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:40,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:40,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:40,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:40,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:40,751 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:40,751 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:40,759 INFO [Thread-1496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1496. 2024-12-02T05:18:40,759 DEBUG [Thread-1496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:40,759 DEBUG [Thread-1496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:40,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202dabfe84ad850438db0b6570ac83e61ad_ff61f31505788790cd78022602747873 is 50, key is test_row_0/A:col10/1733116720748/Put/seqid=0 2024-12-02T05:18:40,759 DEBUG [Thread-1496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:40,759 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:40,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742230_1406 (size=12454) 2024-12-02T05:18:41,163 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:41,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202dabfe84ad850438db0b6570ac83e61ad_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dabfe84ad850438db0b6570ac83e61ad_ff61f31505788790cd78022602747873 2024-12-02T05:18:41,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/18982c68c23946bab259c7842be99f02, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:41,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/18982c68c23946bab259c7842be99f02 is 175, key is test_row_0/A:col10/1733116720748/Put/seqid=0 2024-12-02T05:18:41,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742231_1407 (size=31255) 2024-12-02T05:18:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T05:18:41,387 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:41,571 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/18982c68c23946bab259c7842be99f02 2024-12-02T05:18:41,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/4dc0ba51fa2d42559187933b00e5f24e is 50, key is test_row_0/B:col10/1733116720748/Put/seqid=0 2024-12-02T05:18:41,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742232_1408 (size=12301) 2024-12-02T05:18:41,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/4dc0ba51fa2d42559187933b00e5f24e 2024-12-02T05:18:41,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/e874f159453f4b7f9781b08e50110c04 is 50, key is test_row_0/C:col10/1733116720748/Put/seqid=0 2024-12-02T05:18:41,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742233_1409 (size=12301) 2024-12-02T05:18:42,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/e874f159453f4b7f9781b08e50110c04 2024-12-02T05:18:42,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/18982c68c23946bab259c7842be99f02 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/18982c68c23946bab259c7842be99f02 2024-12-02T05:18:42,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/18982c68c23946bab259c7842be99f02, entries=150, sequenceid=318, filesize=30.5 K 2024-12-02T05:18:42,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/4dc0ba51fa2d42559187933b00e5f24e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/4dc0ba51fa2d42559187933b00e5f24e 2024-12-02T05:18:42,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/4dc0ba51fa2d42559187933b00e5f24e, entries=150, sequenceid=318, filesize=12.0 K 2024-12-02T05:18:42,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/e874f159453f4b7f9781b08e50110c04 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/e874f159453f4b7f9781b08e50110c04 2024-12-02T05:18:42,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/e874f159453f4b7f9781b08e50110c04, entries=150, sequenceid=318, filesize=12.0 K 2024-12-02T05:18:42,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=6.71 KB/6870 for ff61f31505788790cd78022602747873 in 1672ms, sequenceid=318, compaction requested=true 2024-12-02T05:18:42,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:42,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:18:42,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:42,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:18:42,422 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:42,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:42,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff61f31505788790cd78022602747873:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:18:42,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:42,422 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:42,423 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:42,423 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:42,423 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/B is initiating minor compaction (all files) 2024-12-02T05:18:42,423 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/A is initiating minor compaction (all files) 2024-12-02T05:18:42,423 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/B in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:42,423 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/A in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:42,423 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/0289e486e10e417bac964c001dc66073, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/62c702fd898b4e8abbc8927e7d524bfa, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/18982c68c23946bab259c7842be99f02] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=92.2 K 2024-12-02T05:18:42,423 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/6de396be7dd3481683112f7308980310, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/76f462efbab641eca262e97981e026f1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/4dc0ba51fa2d42559187933b00e5f24e] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=36.6 K 2024-12-02T05:18:42,423 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:42,423 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/0289e486e10e417bac964c001dc66073, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/62c702fd898b4e8abbc8927e7d524bfa, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/18982c68c23946bab259c7842be99f02] 2024-12-02T05:18:42,423 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 6de396be7dd3481683112f7308980310, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733116716540 2024-12-02T05:18:42,423 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0289e486e10e417bac964c001dc66073, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733116716540 2024-12-02T05:18:42,424 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 76f462efbab641eca262e97981e026f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733116717238 2024-12-02T05:18:42,424 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62c702fd898b4e8abbc8927e7d524bfa, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733116717238 2024-12-02T05:18:42,424 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18982c68c23946bab259c7842be99f02, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733116718526 2024-12-02T05:18:42,424 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dc0ba51fa2d42559187933b00e5f24e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733116718526 2024-12-02T05:18:42,432 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#B#compaction#344 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:42,432 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/d539ce28941d49a598183854d73594c9 is 50, key is test_row_0/B:col10/1733116720748/Put/seqid=0 2024-12-02T05:18:42,433 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:42,435 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412024477d281d5654d328968c8554cfd2590_ff61f31505788790cd78022602747873 store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:42,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742234_1410 (size=12404) 2024-12-02T05:18:42,449 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412024477d281d5654d328968c8554cfd2590_ff61f31505788790cd78022602747873, store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:42,449 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024477d281d5654d328968c8554cfd2590_ff61f31505788790cd78022602747873 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:42,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742235_1411 (size=4469) 2024-12-02T05:18:42,840 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/d539ce28941d49a598183854d73594c9 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/d539ce28941d49a598183854d73594c9 2024-12-02T05:18:42,843 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff61f31505788790cd78022602747873/B of ff61f31505788790cd78022602747873 into d539ce28941d49a598183854d73594c9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:42,843 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:42,843 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/B, priority=13, startTime=1733116722422; duration=0sec 2024-12-02T05:18:42,843 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:42,843 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:B 2024-12-02T05:18:42,843 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:42,844 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:42,844 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): ff61f31505788790cd78022602747873/C is initiating minor compaction (all files) 2024-12-02T05:18:42,844 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff61f31505788790cd78022602747873/C in TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:42,844 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/830d210758e84d75830b610b40ca697a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/a474e964b3b44819a57a04e4be965858, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/e874f159453f4b7f9781b08e50110c04] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp, totalSize=36.6 K 2024-12-02T05:18:42,844 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 830d210758e84d75830b610b40ca697a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733116716540 2024-12-02T05:18:42,845 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting a474e964b3b44819a57a04e4be965858, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733116717238 2024-12-02T05:18:42,845 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting e874f159453f4b7f9781b08e50110c04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733116718526 2024-12-02T05:18:42,850 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#C#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:42,851 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/43ac413c6cbb4383bc94dd37db885baf is 50, key is test_row_0/C:col10/1733116720748/Put/seqid=0 2024-12-02T05:18:42,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742236_1412 (size=12404) 2024-12-02T05:18:42,854 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff61f31505788790cd78022602747873#A#compaction#345 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:42,855 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/878dfc6c49974678b3bf1eedd2c7ba57 is 175, key is test_row_0/A:col10/1733116720748/Put/seqid=0 2024-12-02T05:18:42,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742237_1413 (size=31358) 2024-12-02T05:18:43,262 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/43ac413c6cbb4383bc94dd37db885baf as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/43ac413c6cbb4383bc94dd37db885baf 2024-12-02T05:18:43,264 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/878dfc6c49974678b3bf1eedd2c7ba57 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/878dfc6c49974678b3bf1eedd2c7ba57 2024-12-02T05:18:43,267 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff61f31505788790cd78022602747873/C of ff61f31505788790cd78022602747873 into 43ac413c6cbb4383bc94dd37db885baf(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:43,267 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:43,267 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/C, priority=13, startTime=1733116722422; duration=0sec 2024-12-02T05:18:43,267 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:43,267 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:C 2024-12-02T05:18:43,268 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff61f31505788790cd78022602747873/A of ff61f31505788790cd78022602747873 into 878dfc6c49974678b3bf1eedd2c7ba57(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:43,268 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff61f31505788790cd78022602747873: 2024-12-02T05:18:43,268 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., storeName=ff61f31505788790cd78022602747873/A, priority=13, startTime=1733116722422; duration=0sec 2024-12-02T05:18:43,268 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:43,268 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff61f31505788790cd78022602747873:A 2024-12-02T05:18:46,781 INFO [Thread-1494 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1494. 2024-12-02T05:18:46,781 DEBUG [Thread-1494 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:46,781 DEBUG [Thread-1494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:46,781 DEBUG [Thread-1494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:46,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:46,809 INFO [Thread-1500 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1500. 2024-12-02T05:18:46,809 DEBUG [Thread-1500 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:46,809 DEBUG [Thread-1500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:46,809 DEBUG [Thread-1500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:46,809 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:46,819 INFO [Thread-1502 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1502. 2024-12-02T05:18:46,819 DEBUG [Thread-1502 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:18:46,819 DEBUG [Thread-1502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:46,819 DEBUG [Thread-1502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:46,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1458 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4369 rows 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1450 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4350 rows 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1462 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4385 rows 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1459 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4370 rows 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1459 2024-12-02T05:18:46,820 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4374 rows 2024-12-02T05:18:46,820 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T05:18:46,820 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T05:18:46,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:46,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:46,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:18:46,821 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T05:18:46,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:46,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T05:18:46,828 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116726828"}]},"ts":"1733116726828"} 2024-12-02T05:18:46,829 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T05:18:46,829 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T05:18:46,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T05:18:46,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff61f31505788790cd78022602747873, UNASSIGN}] 2024-12-02T05:18:46,831 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff61f31505788790cd78022602747873, UNASSIGN 2024-12-02T05:18:46,832 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=ff61f31505788790cd78022602747873, regionState=CLOSING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:18:46,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff61f31505788790cd78022602747873, UNASSIGN because future has completed 2024-12-02T05:18:46,833 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T05:18:46,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:18:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T05:18:46,985 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close ff61f31505788790cd78022602747873 2024-12-02T05:18:46,985 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T05:18:46,985 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing ff61f31505788790cd78022602747873, disabling compactions & flushes 2024-12-02T05:18:46,986 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:46,986 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:46,986 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. after waiting 0 ms 2024-12-02T05:18:46,986 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:46,986 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing ff61f31505788790cd78022602747873 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-02T05:18:46,986 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=A 2024-12-02T05:18:46,986 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:46,986 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=B 2024-12-02T05:18:46,986 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:46,986 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ff61f31505788790cd78022602747873, store=C 2024-12-02T05:18:46,986 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:46,991 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120225fcc3cc0a2e4a2da8d9f1fb92a63db6_ff61f31505788790cd78022602747873 is 50, key is test_row_1/A:col10/1733116726818/Put/seqid=0 2024-12-02T05:18:46,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742238_1414 (size=9914) 2024-12-02T05:18:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T05:18:47,395 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:47,399 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120225fcc3cc0a2e4a2da8d9f1fb92a63db6_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120225fcc3cc0a2e4a2da8d9f1fb92a63db6_ff61f31505788790cd78022602747873 2024-12-02T05:18:47,399 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/79b5a52e21d2418b8ab29d38cfd42350, store: [table=TestAcidGuarantees family=A region=ff61f31505788790cd78022602747873] 2024-12-02T05:18:47,400 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/79b5a52e21d2418b8ab29d38cfd42350 is 175, key is test_row_1/A:col10/1733116726818/Put/seqid=0 2024-12-02T05:18:47,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742239_1415 (size=22561) 2024-12-02T05:18:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T05:18:47,804 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/79b5a52e21d2418b8ab29d38cfd42350 2024-12-02T05:18:47,815 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/d7e8fca143c74848b64c5a2965215f55 is 50, key is test_row_1/B:col10/1733116726818/Put/seqid=0 2024-12-02T05:18:47,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742240_1416 (size=9857) 2024-12-02T05:18:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T05:18:48,220 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/d7e8fca143c74848b64c5a2965215f55 2024-12-02T05:18:48,234 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/f47d6c6db15c4a189b0cf2019e24f91a is 50, key is test_row_1/C:col10/1733116726818/Put/seqid=0 2024-12-02T05:18:48,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742241_1417 (size=9857) 2024-12-02T05:18:48,639 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/f47d6c6db15c4a189b0cf2019e24f91a 2024-12-02T05:18:48,647 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/A/79b5a52e21d2418b8ab29d38cfd42350 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/79b5a52e21d2418b8ab29d38cfd42350 2024-12-02T05:18:48,653 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/79b5a52e21d2418b8ab29d38cfd42350, entries=100, sequenceid=328, filesize=22.0 K 2024-12-02T05:18:48,654 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/B/d7e8fca143c74848b64c5a2965215f55 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/d7e8fca143c74848b64c5a2965215f55 2024-12-02T05:18:48,659 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/d7e8fca143c74848b64c5a2965215f55, entries=100, sequenceid=328, filesize=9.6 K 2024-12-02T05:18:48,660 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/.tmp/C/f47d6c6db15c4a189b0cf2019e24f91a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f47d6c6db15c4a189b0cf2019e24f91a 2024-12-02T05:18:48,664 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f47d6c6db15c4a189b0cf2019e24f91a, entries=100, sequenceid=328, filesize=9.6 K 2024-12-02T05:18:48,665 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ff61f31505788790cd78022602747873 in 1679ms, sequenceid=328, compaction requested=false 2024-12-02T05:18:48,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/0289e486e10e417bac964c001dc66073, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/62c702fd898b4e8abbc8927e7d524bfa, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/18982c68c23946bab259c7842be99f02] to archive 2024-12-02T05:18:48,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:18:48,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/0289e486e10e417bac964c001dc66073 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/0289e486e10e417bac964c001dc66073 2024-12-02T05:18:48,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/62c702fd898b4e8abbc8927e7d524bfa to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/62c702fd898b4e8abbc8927e7d524bfa 2024-12-02T05:18:48,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/18982c68c23946bab259c7842be99f02 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/18982c68c23946bab259c7842be99f02 2024-12-02T05:18:48,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/6de396be7dd3481683112f7308980310, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/76f462efbab641eca262e97981e026f1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/4dc0ba51fa2d42559187933b00e5f24e] to archive 2024-12-02T05:18:48,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:18:48,675 DEBUG [StoreCloser-TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/6de396be7dd3481683112f7308980310 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/6de396be7dd3481683112f7308980310 2024-12-02T05:18:48,676 DEBUG [StoreCloser-TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/76f462efbab641eca262e97981e026f1 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/76f462efbab641eca262e97981e026f1 2024-12-02T05:18:48,678 DEBUG [StoreCloser-TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/4dc0ba51fa2d42559187933b00e5f24e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/4dc0ba51fa2d42559187933b00e5f24e 2024-12-02T05:18:48,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/830d210758e84d75830b610b40ca697a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/a474e964b3b44819a57a04e4be965858, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/e874f159453f4b7f9781b08e50110c04] to archive 2024-12-02T05:18:48,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:18:48,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/830d210758e84d75830b610b40ca697a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/830d210758e84d75830b610b40ca697a 2024-12-02T05:18:48,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/a474e964b3b44819a57a04e4be965858 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/a474e964b3b44819a57a04e4be965858 2024-12-02T05:18:48,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/e874f159453f4b7f9781b08e50110c04 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/e874f159453f4b7f9781b08e50110c04 2024-12-02T05:18:48,689 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/recovered.edits/331.seqid, newMaxSeqId=331, maxSeqId=4 2024-12-02T05:18:48,690 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. 2024-12-02T05:18:48,690 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for ff61f31505788790cd78022602747873: Waiting for close lock at 1733116726985Running coprocessor pre-close hooks at 1733116726985Disabling compacts and flushes for region at 1733116726985Disabling writes for close at 1733116726986 (+1 ms)Obtaining lock to block concurrent updates at 1733116726986Preparing flush snapshotting stores in ff61f31505788790cd78022602747873 at 1733116726986Finished memstore snapshotting TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733116726986Flushing stores of TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873. at 1733116726986Flushing ff61f31505788790cd78022602747873/A: creating writer at 1733116726987 (+1 ms)Flushing mob file ff61f31505788790cd78022602747873/A: appending metadata at 1733116726991 (+4 ms)Flushing mob file ff61f31505788790cd78022602747873/A: closing flushed file at 1733116726991Flushing ff61f31505788790cd78022602747873/A: appending metadata at 1733116727399 (+408 ms)Flushing ff61f31505788790cd78022602747873/A: closing flushed file at 1733116727400 (+1 ms)Flushing ff61f31505788790cd78022602747873/B: creating writer at 1733116727810 (+410 ms)Flushing ff61f31505788790cd78022602747873/B: appending metadata at 1733116727814 (+4 ms)Flushing ff61f31505788790cd78022602747873/B: closing flushed file at 1733116727814Flushing ff61f31505788790cd78022602747873/C: creating writer at 1733116728230 (+416 ms)Flushing ff61f31505788790cd78022602747873/C: appending metadata at 1733116728233 (+3 ms)Flushing ff61f31505788790cd78022602747873/C: closing flushed file at 1733116728233Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fea1038: reopening flushed file at 1733116728646 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@592cddbb: reopening flushed file at 1733116728653 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49adfa87: reopening flushed file at 1733116728659 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ff61f31505788790cd78022602747873 in 1679ms, sequenceid=328, compaction requested=false at 1733116728665 (+6 ms)Writing region close event to WAL at 1733116728686 (+21 ms)Running coprocessor post-close hooks at 1733116728690 (+4 ms)Closed at 1733116728690 2024-12-02T05:18:48,693 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed ff61f31505788790cd78022602747873 2024-12-02T05:18:48,693 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=ff61f31505788790cd78022602747873, regionState=CLOSED 2024-12-02T05:18:48,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:18:48,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-02T05:18:48,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure ff61f31505788790cd78022602747873, server=f49461030c22,46495,1733116594459 in 1.8630 sec 2024-12-02T05:18:48,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-02T05:18:48,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff61f31505788790cd78022602747873, UNASSIGN in 1.8680 sec 2024-12-02T05:18:48,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-02T05:18:48,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8720 sec 2024-12-02T05:18:48,706 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116728706"}]},"ts":"1733116728706"} 2024-12-02T05:18:48,708 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T05:18:48,708 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T05:18:48,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8880 sec 2024-12-02T05:18:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T05:18:48,978 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:48,979 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T05:18:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:48,983 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T05:18:48,984 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:48,988 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873 2024-12-02T05:18:48,991 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/recovered.edits] 2024-12-02T05:18:48,995 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/79b5a52e21d2418b8ab29d38cfd42350 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/79b5a52e21d2418b8ab29d38cfd42350 2024-12-02T05:18:48,996 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/878dfc6c49974678b3bf1eedd2c7ba57 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/A/878dfc6c49974678b3bf1eedd2c7ba57 2024-12-02T05:18:48,998 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/d539ce28941d49a598183854d73594c9 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/d539ce28941d49a598183854d73594c9 2024-12-02T05:18:48,999 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/d7e8fca143c74848b64c5a2965215f55 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/B/d7e8fca143c74848b64c5a2965215f55 2024-12-02T05:18:49,001 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/43ac413c6cbb4383bc94dd37db885baf to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/43ac413c6cbb4383bc94dd37db885baf 2024-12-02T05:18:49,002 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f47d6c6db15c4a189b0cf2019e24f91a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/C/f47d6c6db15c4a189b0cf2019e24f91a 2024-12-02T05:18:49,004 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/recovered.edits/331.seqid to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873/recovered.edits/331.seqid 2024-12-02T05:18:49,005 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/ff61f31505788790cd78022602747873 2024-12-02T05:18:49,005 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T05:18:49,006 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T05:18:49,006 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T05:18:49,010 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020868bdc8782a4ef9bfde85d4144a8b29_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020868bdc8782a4ef9bfde85d4144a8b29_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,012 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020c57c6086cc5450dbbf1a38887ecd0e4_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020c57c6086cc5450dbbf1a38887ecd0e4_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,014 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120225fcc3cc0a2e4a2da8d9f1fb92a63db6_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120225fcc3cc0a2e4a2da8d9f1fb92a63db6_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,015 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202280bb4acbaa34dab850d9da0f4fd5af8_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202280bb4acbaa34dab850d9da0f4fd5af8_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,017 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120245845108a2a1457fb87cd490e5a822ac_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120245845108a2a1457fb87cd490e5a822ac_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,018 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026045f53963cf49f88a77d4fe918c7147_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026045f53963cf49f88a77d4fe918c7147_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,019 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026b632919d6a4477f86c69bb8c2889a54_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026b632919d6a4477f86c69bb8c2889a54_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,021 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120273ba079109f44008af5c52e80eb5825d_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120273ba079109f44008af5c52e80eb5825d_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,022 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029ae0656e17f24d5684f32806a1fc5ee7_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029ae0656e17f24d5684f32806a1fc5ee7_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,024 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b69f621965a54fdfbaf9bfe399f36ab4_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b69f621965a54fdfbaf9bfe399f36ab4_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,025 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ba5eb8cfd7d24ed096c0aa6e2b9896f2_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ba5eb8cfd7d24ed096c0aa6e2b9896f2_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,027 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bf3c7ea75c0444e1ac52b113c1b0b48d_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bf3c7ea75c0444e1ac52b113c1b0b48d_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,028 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c63200a738284769a7964122c388d4a4_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c63200a738284769a7964122c388d4a4_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,029 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d4af9eee4b354252b8521eb787046f7e_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d4af9eee4b354252b8521eb787046f7e_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,030 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dabfe84ad850438db0b6570ac83e61ad_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dabfe84ad850438db0b6570ac83e61ad_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,031 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e3831dc138614da9b1ef38ae4f0563e3_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e3831dc138614da9b1ef38ae4f0563e3_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,032 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202edca0c401a48456fa9aceac2491079f7_ff61f31505788790cd78022602747873 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202edca0c401a48456fa9aceac2491079f7_ff61f31505788790cd78022602747873 2024-12-02T05:18:49,032 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T05:18:49,034 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:49,036 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T05:18:49,037 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T05:18:49,038 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:49,038 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T05:18:49,038 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733116729038"}]},"ts":"9223372036854775807"} 2024-12-02T05:18:49,040 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T05:18:49,040 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ff61f31505788790cd78022602747873, NAME => 'TestAcidGuarantees,,1733116697125.ff61f31505788790cd78022602747873.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T05:18:49,040 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T05:18:49,040 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733116729040"}]},"ts":"9223372036854775807"} 2024-12-02T05:18:49,042 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T05:18:49,042 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:49,043 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-12-02T05:18:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T05:18:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T05:18:49,093 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:49,102 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 246) - Thread LEAK? -, OpenFileDescriptor=443 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=669 (was 574) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5821 (was 6361) 2024-12-02T05:18:49,111 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=669, ProcessCount=11, AvailableMemoryMB=5821 2024-12-02T05:18:49,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T05:18:49,113 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T05:18:49,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T05:18:49,115 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T05:18:49,115 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:49,115 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-12-02T05:18:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-02T05:18:49,116 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T05:18:49,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742242_1418 (size=1061) 2024-12-02T05:18:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-02T05:18:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-02T05:18:49,524 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c 2024-12-02T05:18:49,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742243_1419 (size=53) 2024-12-02T05:18:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-02T05:18:49,933 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:18:49,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing be075553e9fdd4e7a03804f307b7c3f9, disabling compactions & flushes 2024-12-02T05:18:49,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:49,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:49,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. after waiting 0 ms 2024-12-02T05:18:49,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:49,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:49,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for be075553e9fdd4e7a03804f307b7c3f9: Waiting for close lock at 1733116729934Disabling compacts and flushes for region at 1733116729934Disabling writes for close at 1733116729934Writing region close event to WAL at 1733116729934Closed at 1733116729934 2024-12-02T05:18:49,936 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T05:18:49,936 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733116729936"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733116729936"}]},"ts":"1733116729936"} 2024-12-02T05:18:49,939 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T05:18:49,940 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T05:18:49,941 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116729940"}]},"ts":"1733116729940"} 2024-12-02T05:18:49,942 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T05:18:49,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be075553e9fdd4e7a03804f307b7c3f9, ASSIGN}] 2024-12-02T05:18:49,944 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be075553e9fdd4e7a03804f307b7c3f9, ASSIGN 2024-12-02T05:18:49,945 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=be075553e9fdd4e7a03804f307b7c3f9, ASSIGN; state=OFFLINE, location=f49461030c22,46495,1733116594459; forceNewPlan=false, retain=false 2024-12-02T05:18:50,096 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=be075553e9fdd4e7a03804f307b7c3f9, regionState=OPENING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:18:50,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=be075553e9fdd4e7a03804f307b7c3f9, ASSIGN because future has completed 2024-12-02T05:18:50,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:18:50,257 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:50,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-02T05:18:50,258 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} 2024-12-02T05:18:50,259 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:50,259 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:18:50,259 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:50,259 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:50,261 INFO [StoreOpener-be075553e9fdd4e7a03804f307b7c3f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:50,263 INFO [StoreOpener-be075553e9fdd4e7a03804f307b7c3f9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:18:50,263 INFO [StoreOpener-be075553e9fdd4e7a03804f307b7c3f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be075553e9fdd4e7a03804f307b7c3f9 columnFamilyName A 2024-12-02T05:18:50,263 DEBUG [StoreOpener-be075553e9fdd4e7a03804f307b7c3f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:50,264 INFO [StoreOpener-be075553e9fdd4e7a03804f307b7c3f9-1 {}] regionserver.HStore(327): Store=be075553e9fdd4e7a03804f307b7c3f9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:18:50,264 INFO [StoreOpener-be075553e9fdd4e7a03804f307b7c3f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:50,265 INFO [StoreOpener-be075553e9fdd4e7a03804f307b7c3f9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:18:50,266 INFO [StoreOpener-be075553e9fdd4e7a03804f307b7c3f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be075553e9fdd4e7a03804f307b7c3f9 columnFamilyName B 2024-12-02T05:18:50,266 DEBUG [StoreOpener-be075553e9fdd4e7a03804f307b7c3f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:50,266 INFO [StoreOpener-be075553e9fdd4e7a03804f307b7c3f9-1 {}] regionserver.HStore(327): Store=be075553e9fdd4e7a03804f307b7c3f9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:18:50,267 INFO [StoreOpener-be075553e9fdd4e7a03804f307b7c3f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:50,268 INFO [StoreOpener-be075553e9fdd4e7a03804f307b7c3f9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:18:50,268 INFO [StoreOpener-be075553e9fdd4e7a03804f307b7c3f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be075553e9fdd4e7a03804f307b7c3f9 columnFamilyName C 2024-12-02T05:18:50,268 DEBUG [StoreOpener-be075553e9fdd4e7a03804f307b7c3f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:18:50,269 INFO [StoreOpener-be075553e9fdd4e7a03804f307b7c3f9-1 {}] regionserver.HStore(327): Store=be075553e9fdd4e7a03804f307b7c3f9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:18:50,269 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:50,269 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:50,270 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:50,271 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:50,272 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:50,272 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:50,273 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T05:18:50,274 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:50,276 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T05:18:50,276 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened be075553e9fdd4e7a03804f307b7c3f9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72378314, jitterRate=0.07852092385292053}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T05:18:50,276 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:50,277 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for be075553e9fdd4e7a03804f307b7c3f9: Running coprocessor pre-open hook at 1733116730259Writing region info on filesystem at 1733116730259Initializing all the Stores at 1733116730260 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116730260Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116730261 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116730261Cleaning up temporary data from old regions at 1733116730272 (+11 ms)Running coprocessor post-open hooks at 1733116730276 (+4 ms)Region opened successfully at 1733116730277 (+1 ms) 2024-12-02T05:18:50,278 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., pid=120, masterSystemTime=1733116730252 2024-12-02T05:18:50,280 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:50,280 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:50,281 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=be075553e9fdd4e7a03804f307b7c3f9, regionState=OPEN, openSeqNum=2, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:18:50,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:18:50,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-02T05:18:50,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 in 184 msec 2024-12-02T05:18:50,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-12-02T05:18:50,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be075553e9fdd4e7a03804f307b7c3f9, ASSIGN in 343 msec 2024-12-02T05:18:50,287 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T05:18:50,287 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116730287"}]},"ts":"1733116730287"} 2024-12-02T05:18:50,289 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T05:18:50,289 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T05:18:50,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-12-02T05:18:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-02T05:18:51,267 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:51,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ad78a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:51,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:51,269 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40826641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:51,271 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f145a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,272 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:51,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:51,275 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T05:18:51,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is f49461030c22,42731,1733116593060 2024-12-02T05:18:51,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@22685265 2024-12-02T05:18:51,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T05:18:51,277 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T05:18:51,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,278 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:51,278 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:51,279 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33817105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:51,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:51,280 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:51,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c14f37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,281 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:51,281 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:51,282 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9870ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:51,283 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@716548fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,284 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:51,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:51,285 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@358c3f1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:51,286 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b6f139a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,287 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:51,287 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:51,287 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4586f0c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:51,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:51,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:51,288 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:51,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40a74aa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:51,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:51,290 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:51,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:51,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:51,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28fbdf8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:51,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:51,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:51,291 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:51,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe85d26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:51,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:51,292 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77835658, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:51,294 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:51,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@753ef8f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,295 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:51,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:51,295 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:51,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:51,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@190b69bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:51,296 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62807b1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,297 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:51,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:51,298 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@294d7adf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:51,299 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6f1055, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:51,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:51,300 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffe5c0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:51,302 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@abf0442, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df5a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:18:51,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:18:51,303 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:18:51,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:18:51,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:18:51,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c180403, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:18:51,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:18:51,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:18:51,304 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:18:51,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@809e4bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:18:51,305 DEBUG [Thread-1901 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:51,305 DEBUG [Thread-1912 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:51,305 DEBUG [Thread-1905 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:51,306 DEBUG [Thread-1907 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:51,306 DEBUG [Thread-1918 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:51,306 DEBUG [Thread-1916 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:51,306 DEBUG [Thread-1914 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:51,306 DEBUG [Thread-1899 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:51,307 DEBUG [Thread-1903 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:51,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:51,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:51,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:51,308 DEBUG [Thread-1910 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:18:51,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:51,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:18:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:18:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:18:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:51,311 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:18:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:18:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:18:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:18:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-02T05:18:51,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T05:18:51,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:18:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:18:51,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:51,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:51,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:18:51,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:18:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:18:51,317 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:18:51,318 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:18:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:18:51,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:18:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:51,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T05:18:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:18:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:18:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:51,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2] 2024-12-02T05:18:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:18:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:51,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/04a49e755b8649c596627a79f3c0bbc1 is 50, key is test_row_0/A:col10/1733116731317/Put/seqid=0 2024-12-02T05:18:51,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733116731346, max timestamp=1733116731359Num uniques 859; 2024-12-02T05:18:51,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T05:18:51,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733116731346, max timestamp=1733116731359Num uniques 859; 2024-12-02T05:18:51,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T05:18:51,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733116731346, max timestamp=1733116731359Num uniques 859; 2024-12-02T05:18:51,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T05:18:51,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742244_1420 (size=12001) 2024-12-02T05:18:51,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/04a49e755b8649c596627a79f3c0bbc1 2024-12-02T05:18:51,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116791390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:51,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:51,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116791391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:51,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:51,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:51,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/3f65037b655a4b81abc608212dcbb9a4 is 50, key is test_row_0/B:col10/1733116731317/Put/seqid=0 2024-12-02T05:18:51,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742245_1421 (size=12001) 2024-12-02T05:18:51,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116791404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:51,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:51,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116791404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:51,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:51,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:51,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116791405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:51,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T05:18:51,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T05:18:51,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:51,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:51,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:51,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:51,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:51,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116791507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116791507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:51,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116791518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:51,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:51,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116791518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:51,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:51,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116791518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:51,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T05:18:51,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:51,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:51,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T05:18:51,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:51,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:51,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116791718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:51,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116791720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:51,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116791728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:51,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116791728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:51,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:51,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:51,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116791737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:51,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T05:18:51,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:51,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:51,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:51,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:51,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:51,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/3f65037b655a4b81abc608212dcbb9a4 2024-12-02T05:18:51,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/cac2948fc2ff4ceab51ceb572661c892 is 50, key is test_row_0/C:col10/1733116731317/Put/seqid=0 2024-12-02T05:18:51,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742246_1422 (size=12001) 2024-12-02T05:18:51,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/cac2948fc2ff4ceab51ceb572661c892 2024-12-02T05:18:51,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/04a49e755b8649c596627a79f3c0bbc1 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/04a49e755b8649c596627a79f3c0bbc1 2024-12-02T05:18:51,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/04a49e755b8649c596627a79f3c0bbc1, entries=150, sequenceid=11, filesize=11.7 K 2024-12-02T05:18:51,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/3f65037b655a4b81abc608212dcbb9a4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/3f65037b655a4b81abc608212dcbb9a4 2024-12-02T05:18:51,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/3f65037b655a4b81abc608212dcbb9a4, entries=150, sequenceid=11, filesize=11.7 K 2024-12-02T05:18:51,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/cac2948fc2ff4ceab51ceb572661c892 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/cac2948fc2ff4ceab51ceb572661c892 2024-12-02T05:18:51,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/cac2948fc2ff4ceab51ceb572661c892, entries=150, sequenceid=11, filesize=11.7 K 2024-12-02T05:18:51,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for be075553e9fdd4e7a03804f307b7c3f9 in 594ms, sequenceid=11, compaction requested=false 2024-12-02T05:18:51,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T05:18:51,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T05:18:51,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:51,938 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-02T05:18:51,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:18:51,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:51,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:18:51,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:51,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:18:51,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:51,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/f2494aa644eb4f72af51412cfbd75eac is 50, key is test_row_0/A:col10/1733116731384/Put/seqid=0 2024-12-02T05:18:51,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742247_1423 (size=12001) 2024-12-02T05:18:51,980 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/f2494aa644eb4f72af51412cfbd75eac 2024-12-02T05:18:52,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/d7ff732822844ba4bc3ea87720f644a1 is 50, key is test_row_0/B:col10/1733116731384/Put/seqid=0 2024-12-02T05:18:52,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742248_1424 (size=12001) 2024-12-02T05:18:52,014 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/d7ff732822844ba4bc3ea87720f644a1 2024-12-02T05:18:52,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/a84a2ab733dc46a895601cc44c41e4a2 is 50, key is test_row_0/C:col10/1733116731384/Put/seqid=0 2024-12-02T05:18:52,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742249_1425 (size=12001) 2024-12-02T05:18:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:52,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:52,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116792053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116792054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116792056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116792057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116792067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116792167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116792167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116792167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116792177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116792377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116792377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116792377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116792387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,435 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/a84a2ab733dc46a895601cc44c41e4a2 2024-12-02T05:18:52,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/f2494aa644eb4f72af51412cfbd75eac as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f2494aa644eb4f72af51412cfbd75eac 2024-12-02T05:18:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T05:18:52,462 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f2494aa644eb4f72af51412cfbd75eac, entries=150, sequenceid=47, filesize=11.7 K 2024-12-02T05:18:52,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/d7ff732822844ba4bc3ea87720f644a1 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d7ff732822844ba4bc3ea87720f644a1 2024-12-02T05:18:52,471 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d7ff732822844ba4bc3ea87720f644a1, entries=150, sequenceid=47, filesize=11.7 K 2024-12-02T05:18:52,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/a84a2ab733dc46a895601cc44c41e4a2 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/a84a2ab733dc46a895601cc44c41e4a2 2024-12-02T05:18:52,477 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/a84a2ab733dc46a895601cc44c41e4a2, entries=150, sequenceid=47, filesize=11.7 K 2024-12-02T05:18:52,478 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for be075553e9fdd4e7a03804f307b7c3f9 in 540ms, sequenceid=47, compaction requested=false 2024-12-02T05:18:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-02T05:18:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-02T05:18:52,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-02T05:18:52,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1630 sec 2024-12-02T05:18:52,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.1730 sec 2024-12-02T05:18:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:52,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T05:18:52,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:18:52,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:52,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:18:52,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:52,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:18:52,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:52,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/e6d24f68c40e47f088fa57f7558d6437 is 50, key is test_row_0/A:col10/1733116732580/Put/seqid=0 2024-12-02T05:18:52,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742250_1426 (size=14341) 2024-12-02T05:18:52,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/e6d24f68c40e47f088fa57f7558d6437 2024-12-02T05:18:52,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/a956f54c7b8e4095b81bc84a6062ef4f is 50, key is test_row_0/B:col10/1733116732580/Put/seqid=0 2024-12-02T05:18:52,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742251_1427 (size=12001) 2024-12-02T05:18:52,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/a956f54c7b8e4095b81bc84a6062ef4f 2024-12-02T05:18:52,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/f425753dcdd64c20871421a4cfbccecf is 50, key is test_row_0/C:col10/1733116732580/Put/seqid=0 2024-12-02T05:18:52,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742252_1428 (size=12001) 2024-12-02T05:18:52,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733116732677, max timestamp=1733116732704Num uniques 859; 2024-12-02T05:18:52,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:18:52,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733116732677, max timestamp=1733116732704Num uniques 859; 2024-12-02T05:18:52,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:18:52,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733116732677, max timestamp=1733116732704Num uniques 859; 2024-12-02T05:18:52,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:18:52,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116792733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116792739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116792741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116792741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116792742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116792847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116792858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116792858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116792858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116792858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/f425753dcdd64c20871421a4cfbccecf 2024-12-02T05:18:53,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/e6d24f68c40e47f088fa57f7558d6437 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e6d24f68c40e47f088fa57f7558d6437 2024-12-02T05:18:53,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e6d24f68c40e47f088fa57f7558d6437, entries=200, sequenceid=57, filesize=14.0 K 2024-12-02T05:18:53,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/a956f54c7b8e4095b81bc84a6062ef4f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/a956f54c7b8e4095b81bc84a6062ef4f 2024-12-02T05:18:53,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/a956f54c7b8e4095b81bc84a6062ef4f, entries=150, sequenceid=57, filesize=11.7 K 2024-12-02T05:18:53,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/f425753dcdd64c20871421a4cfbccecf as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f425753dcdd64c20871421a4cfbccecf 2024-12-02T05:18:53,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f425753dcdd64c20871421a4cfbccecf, entries=150, sequenceid=57, filesize=11.7 K 2024-12-02T05:18:53,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116793057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for be075553e9fdd4e7a03804f307b7c3f9 in 480ms, sequenceid=57, compaction requested=true 2024-12-02T05:18:53,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:53,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:18:53,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:53,062 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:53,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:18:53,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:53,062 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:53,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:18:53,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:53,063 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:53,063 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/A is initiating minor compaction (all files) 2024-12-02T05:18:53,063 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:53,063 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/A in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:53,063 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/B is initiating minor compaction (all files) 2024-12-02T05:18:53,063 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/B in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:53,063 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/04a49e755b8649c596627a79f3c0bbc1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f2494aa644eb4f72af51412cfbd75eac, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e6d24f68c40e47f088fa57f7558d6437] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=37.4 K 2024-12-02T05:18:53,063 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/3f65037b655a4b81abc608212dcbb9a4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d7ff732822844ba4bc3ea87720f644a1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/a956f54c7b8e4095b81bc84a6062ef4f] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=35.2 K 2024-12-02T05:18:53,064 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04a49e755b8649c596627a79f3c0bbc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733116731315 2024-12-02T05:18:53,064 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f65037b655a4b81abc608212dcbb9a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733116731315 2024-12-02T05:18:53,064 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2494aa644eb4f72af51412cfbd75eac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733116731384 2024-12-02T05:18:53,064 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting d7ff732822844ba4bc3ea87720f644a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733116731384 2024-12-02T05:18:53,064 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6d24f68c40e47f088fa57f7558d6437, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733116732053 2024-12-02T05:18:53,064 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting a956f54c7b8e4095b81bc84a6062ef4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733116732055 2024-12-02T05:18:53,075 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#A#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:53,076 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/8ca0540e4f574add8ccc8d2f21533cda is 50, key is test_row_0/A:col10/1733116732580/Put/seqid=0 2024-12-02T05:18:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:53,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-02T05:18:53,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:18:53,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:53,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:18:53,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:53,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:18:53,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:53,078 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:53,079 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/e6dfdcc09adb48c1b98873e3c89098b2 is 50, key is test_row_0/B:col10/1733116732580/Put/seqid=0 2024-12-02T05:18:53,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/f875551ffc804b2f9b722669b621f1bf is 50, key is test_row_0/A:col10/1733116733073/Put/seqid=0 2024-12-02T05:18:53,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742253_1429 (size=12104) 2024-12-02T05:18:53,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742255_1431 (size=12001) 2024-12-02T05:18:53,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742254_1430 (size=12104) 2024-12-02T05:18:53,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116793091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116793091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116793095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116793102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,106 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/e6dfdcc09adb48c1b98873e3c89098b2 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/e6dfdcc09adb48c1b98873e3c89098b2 2024-12-02T05:18:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,110 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/B of be075553e9fdd4e7a03804f307b7c3f9 into e6dfdcc09adb48c1b98873e3c89098b2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:53,110 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:53,110 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/B, priority=13, startTime=1733116733062; duration=0sec 2024-12-02T05:18:53,110 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:53,110 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:B 2024-12-02T05:18:53,110 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:53,111 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:53,112 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/C is initiating minor compaction (all files) 2024-12-02T05:18:53,112 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/C in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:53,112 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/cac2948fc2ff4ceab51ceb572661c892, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/a84a2ab733dc46a895601cc44c41e4a2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f425753dcdd64c20871421a4cfbccecf] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=35.2 K 2024-12-02T05:18:53,112 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting cac2948fc2ff4ceab51ceb572661c892, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733116731315 2024-12-02T05:18:53,113 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting a84a2ab733dc46a895601cc44c41e4a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733116731384 2024-12-02T05:18:53,113 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting f425753dcdd64c20871421a4cfbccecf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733116732055 2024-12-02T05:18:53,118 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#C#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:53,119 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/d3c71346056b4163ae6be53c07792dd6 is 50, key is test_row_0/C:col10/1733116732580/Put/seqid=0 2024-12-02T05:18:53,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742256_1432 (size=12104) 2024-12-02T05:18:53,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116793207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116793207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116793207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116793218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116793368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116793417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116793417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116793418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116793428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T05:18:53,457 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:53,458 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:18:53,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/f875551ffc804b2f9b722669b621f1bf 2024-12-02T05:18:53,605 INFO [AsyncFSWAL-0-hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData-prefix:f49461030c22,42731,1733116593060 {}] wal.AbstractFSWAL(1368): Slow sync cost: 146 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34519,DS-5a5c0e22-a56e-4f32-ae1e-a90082acf4bf,DISK]] 2024-12-02T05:18:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-02T05:18:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T05:18:53,607 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:18:53,608 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:18:53,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:18:53,615 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/8ca0540e4f574add8ccc8d2f21533cda as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/8ca0540e4f574add8ccc8d2f21533cda 2024-12-02T05:18:53,616 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/d3c71346056b4163ae6be53c07792dd6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/d3c71346056b4163ae6be53c07792dd6 2024-12-02T05:18:53,623 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/A of be075553e9fdd4e7a03804f307b7c3f9 into 8ca0540e4f574add8ccc8d2f21533cda(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:53,623 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:53,623 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/A, priority=13, startTime=1733116733062; duration=0sec 2024-12-02T05:18:53,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/9373723c51c54d6a8d3d69a695a90c4a is 50, key is test_row_0/B:col10/1733116733073/Put/seqid=0 2024-12-02T05:18:53,623 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:53,623 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:A 2024-12-02T05:18:53,628 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/C of be075553e9fdd4e7a03804f307b7c3f9 into d3c71346056b4163ae6be53c07792dd6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:53,628 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:53,628 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/C, priority=13, startTime=1733116733062; duration=0sec 2024-12-02T05:18:53,628 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:53,628 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:C 2024-12-02T05:18:53,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742257_1433 (size=12001) 2024-12-02T05:18:53,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/9373723c51c54d6a8d3d69a695a90c4a 2024-12-02T05:18:53,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/e7b3c05a92ca4b21991b098a2536fab1 is 50, key is test_row_0/C:col10/1733116733073/Put/seqid=0 2024-12-02T05:18:53,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742258_1434 (size=12001) 2024-12-02T05:18:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T05:18:53,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116793727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116793727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116793727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116793737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T05:18:53,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:53,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:53,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:53,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:53,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:53,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116793888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:53,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T05:18:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:53,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T05:18:54,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/e7b3c05a92ca4b21991b098a2536fab1 2024-12-02T05:18:54,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/f875551ffc804b2f9b722669b621f1bf as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f875551ffc804b2f9b722669b621f1bf 2024-12-02T05:18:54,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f875551ffc804b2f9b722669b621f1bf, entries=150, sequenceid=94, filesize=11.7 K 2024-12-02T05:18:54,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/9373723c51c54d6a8d3d69a695a90c4a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/9373723c51c54d6a8d3d69a695a90c4a 2024-12-02T05:18:54,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/9373723c51c54d6a8d3d69a695a90c4a, entries=150, sequenceid=94, filesize=11.7 K 2024-12-02T05:18:54,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/e7b3c05a92ca4b21991b098a2536fab1 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e7b3c05a92ca4b21991b098a2536fab1 2024-12-02T05:18:54,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e7b3c05a92ca4b21991b098a2536fab1, entries=150, sequenceid=94, filesize=11.7 K 2024-12-02T05:18:54,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for be075553e9fdd4e7a03804f307b7c3f9 in 980ms, sequenceid=94, compaction requested=false 2024-12-02T05:18:54,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:54,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T05:18:54,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:54,068 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-02T05:18:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:18:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:18:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:18:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:54,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/e8b90b470c804268b7dc3b9cbfcba2e6 is 50, key is test_row_0/A:col10/1733116733089/Put/seqid=0 2024-12-02T05:18:54,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742259_1435 (size=12001) 2024-12-02T05:18:54,208 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T05:18:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T05:18:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:54,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:54,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733116734252, max timestamp=1733116734264Num uniques 859; 2024-12-02T05:18:54,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T05:18:54,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733116734252, max timestamp=1733116734264Num uniques 859; 2024-12-02T05:18:54,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733116734252, max timestamp=1733116734264Num uniques 859; 2024-12-02T05:18:54,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T05:18:54,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T05:18:54,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116794320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116794321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116794331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116794332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116794437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116794437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116794448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116794448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,493 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/e8b90b470c804268b7dc3b9cbfcba2e6 2024-12-02T05:18:54,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/5103e3d41bd84fd4a7ad5562a88a8622 is 50, key is test_row_0/B:col10/1733116733089/Put/seqid=0 2024-12-02T05:18:54,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742260_1436 (size=12001) 2024-12-02T05:18:54,505 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/5103e3d41bd84fd4a7ad5562a88a8622 2024-12-02T05:18:54,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/a9821ff770df4855bb394359654ced06 is 50, key is test_row_0/C:col10/1733116733089/Put/seqid=0 2024-12-02T05:18:54,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742261_1437 (size=12001) 2024-12-02T05:18:54,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116794647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116794647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116794658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116794658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T05:18:54,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116794907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,915 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in be075553e9fdd4e7a03804f307b7c3f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:18:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,925 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/a9821ff770df4855bb394359654ced06 2024-12-02T05:18:54,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/e8b90b470c804268b7dc3b9cbfcba2e6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e8b90b470c804268b7dc3b9cbfcba2e6 2024-12-02T05:18:54,932 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e8b90b470c804268b7dc3b9cbfcba2e6, entries=150, sequenceid=106, filesize=11.7 K 2024-12-02T05:18:54,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/5103e3d41bd84fd4a7ad5562a88a8622 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/5103e3d41bd84fd4a7ad5562a88a8622 2024-12-02T05:18:54,937 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/5103e3d41bd84fd4a7ad5562a88a8622, entries=150, sequenceid=106, filesize=11.7 K 2024-12-02T05:18:54,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/a9821ff770df4855bb394359654ced06 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/a9821ff770df4855bb394359654ced06 2024-12-02T05:18:54,941 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/a9821ff770df4855bb394359654ced06, entries=150, sequenceid=106, filesize=11.7 K 2024-12-02T05:18:54,941 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for be075553e9fdd4e7a03804f307b7c3f9 in 873ms, sequenceid=106, compaction requested=true 2024-12-02T05:18:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:54,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-02T05:18:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-02T05:18:54,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-02T05:18:54,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3350 sec 2024-12-02T05:18:54,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.4880 sec 2024-12-02T05:18:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:54,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-12-02T05:18:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:18:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T05:18:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:18:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T05:18:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:18:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T05:18:54,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/7dc7688e9db94b99b1b89a5b4a5e6d85 is 50, key is test_row_0/A:col10/1733116734293/Put/seqid=0 2024-12-02T05:18:54,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742262_1438 (size=16931) 2024-12-02T05:18:54,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116794985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116794986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116794987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116794988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:55,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116795097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:55,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116795107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116795108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:55,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:55,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116795108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:55,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:55,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116795317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:55,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116795327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:55,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:55,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116795327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:55,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:55,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:55,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116795327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:55,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:55,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:55,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/7dc7688e9db94b99b1b89a5b4a5e6d85 2024-12-02T05:18:55,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/38795e78f5a642479923181781c2ff48 is 50, key is test_row_0/B:col10/1733116734293/Put/seqid=0 2024-12-02T05:18:55,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742263_1439 (size=12151) 2024-12-02T05:18:55,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116795627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:55,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116795637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:55,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116795637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116795637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T05:18:55,758 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:55,758 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:18:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-02T05:18:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T05:18:55,760 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:18:55,760 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:18:55,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:18:55,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/38795e78f5a642479923181781c2ff48 2024-12-02T05:18:55,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/e586ddd19d474d4a9455252b6f5493aa is 50, key is test_row_0/C:col10/1733116734293/Put/seqid=0 2024-12-02T05:18:55,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742264_1440 (size=12151) 2024-12-02T05:18:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T05:18:55,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T05:18:55,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:55,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:55,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:55,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:55,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:56,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T05:18:56,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:56,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:56,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:56,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:56,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T05:18:56,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116796137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:56,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116796147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:56,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116796158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:56,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116796158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:56,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/e586ddd19d474d4a9455252b6f5493aa 2024-12-02T05:18:56,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/7dc7688e9db94b99b1b89a5b4a5e6d85 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/7dc7688e9db94b99b1b89a5b4a5e6d85 2024-12-02T05:18:56,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T05:18:56,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:56,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:56,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:56,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:56,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:56,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/7dc7688e9db94b99b1b89a5b4a5e6d85, entries=250, sequenceid=144, filesize=16.5 K 2024-12-02T05:18:56,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/38795e78f5a642479923181781c2ff48 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/38795e78f5a642479923181781c2ff48 2024-12-02T05:18:56,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/38795e78f5a642479923181781c2ff48, entries=150, sequenceid=144, filesize=11.9 K 2024-12-02T05:18:56,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/e586ddd19d474d4a9455252b6f5493aa as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e586ddd19d474d4a9455252b6f5493aa 2024-12-02T05:18:56,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e586ddd19d474d4a9455252b6f5493aa, entries=150, sequenceid=144, filesize=11.9 K 2024-12-02T05:18:56,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~451.27 KB/462096, currentSize=33.54 KB/34350 for be075553e9fdd4e7a03804f307b7c3f9 in 1254ms, sequenceid=144, compaction requested=true 2024-12-02T05:18:56,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:18:56,228 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:18:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:56,228 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:18:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:18:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:18:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:56,229 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53037 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:18:56,229 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:18:56,229 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/A is initiating minor compaction (all files) 2024-12-02T05:18:56,229 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/B is initiating minor compaction (all files) 2024-12-02T05:18:56,229 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/A in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:56,229 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/B in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:56,229 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/8ca0540e4f574add8ccc8d2f21533cda, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f875551ffc804b2f9b722669b621f1bf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e8b90b470c804268b7dc3b9cbfcba2e6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/7dc7688e9db94b99b1b89a5b4a5e6d85] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=51.8 K 2024-12-02T05:18:56,229 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/e6dfdcc09adb48c1b98873e3c89098b2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/9373723c51c54d6a8d3d69a695a90c4a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/5103e3d41bd84fd4a7ad5562a88a8622, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/38795e78f5a642479923181781c2ff48] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=47.1 K 2024-12-02T05:18:56,229 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ca0540e4f574add8ccc8d2f21533cda, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733116732055 2024-12-02T05:18:56,229 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting e6dfdcc09adb48c1b98873e3c89098b2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733116732055 2024-12-02T05:18:56,229 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting f875551ffc804b2f9b722669b621f1bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733116732738 2024-12-02T05:18:56,229 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 9373723c51c54d6a8d3d69a695a90c4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733116732738 2024-12-02T05:18:56,230 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 5103e3d41bd84fd4a7ad5562a88a8622, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733116733089 2024-12-02T05:18:56,230 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8b90b470c804268b7dc3b9cbfcba2e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733116733089 2024-12-02T05:18:56,230 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dc7688e9db94b99b1b89a5b4a5e6d85, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733116734293 2024-12-02T05:18:56,230 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 38795e78f5a642479923181781c2ff48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733116734293 2024-12-02T05:18:56,247 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#A#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:56,247 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/f791d6f905fb4ab7a882652aaa287f84 is 50, key is test_row_0/A:col10/1733116734293/Put/seqid=0 2024-12-02T05:18:56,249 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#B#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:56,250 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/9cce3c1b62e44b3f85393670868af6be is 50, key is test_row_0/B:col10/1733116734293/Put/seqid=0 2024-12-02T05:18:56,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742265_1441 (size=12391) 2024-12-02T05:18:56,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742266_1442 (size=12391) 2024-12-02T05:18:56,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T05:18:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:56,371 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T05:18:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:18:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:18:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:18:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:56,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/9124051fc45945ffad8d8a18040d42dd is 50, key is test_row_1/A:col10/1733116734986/Put/seqid=0 2024-12-02T05:18:56,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742267_1443 (size=9757) 2024-12-02T05:18:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T05:18:56,665 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/f791d6f905fb4ab7a882652aaa287f84 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f791d6f905fb4ab7a882652aaa287f84 2024-12-02T05:18:56,665 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/9cce3c1b62e44b3f85393670868af6be as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/9cce3c1b62e44b3f85393670868af6be 2024-12-02T05:18:56,670 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/A of be075553e9fdd4e7a03804f307b7c3f9 into f791d6f905fb4ab7a882652aaa287f84(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:56,670 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:56,670 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/A, priority=12, startTime=1733116736228; duration=0sec 2024-12-02T05:18:56,670 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:56,670 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:A 2024-12-02T05:18:56,670 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:18:56,671 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/B of be075553e9fdd4e7a03804f307b7c3f9 into 9cce3c1b62e44b3f85393670868af6be(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:56,671 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:56,671 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/B, priority=12, startTime=1733116736228; duration=0sec 2024-12-02T05:18:56,671 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:56,671 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:B 2024-12-02T05:18:56,671 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:18:56,671 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/C is initiating minor compaction (all files) 2024-12-02T05:18:56,672 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/C in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:56,672 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/d3c71346056b4163ae6be53c07792dd6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e7b3c05a92ca4b21991b098a2536fab1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/a9821ff770df4855bb394359654ced06, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e586ddd19d474d4a9455252b6f5493aa] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=47.1 K 2024-12-02T05:18:56,672 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3c71346056b4163ae6be53c07792dd6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733116732055 2024-12-02T05:18:56,672 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7b3c05a92ca4b21991b098a2536fab1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733116732738 2024-12-02T05:18:56,672 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9821ff770df4855bb394359654ced06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733116733089 2024-12-02T05:18:56,673 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting e586ddd19d474d4a9455252b6f5493aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733116734293 2024-12-02T05:18:56,683 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#C#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:56,683 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/f8956066bb804c52b3b367fb018c2473 is 50, key is test_row_0/C:col10/1733116734293/Put/seqid=0 2024-12-02T05:18:56,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742268_1444 (size=12391) 2024-12-02T05:18:56,779 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/9124051fc45945ffad8d8a18040d42dd 2024-12-02T05:18:56,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/278c53564cfe43b9bfd6c1742178d85c is 50, key is test_row_1/B:col10/1733116734986/Put/seqid=0 2024-12-02T05:18:56,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742269_1445 (size=9757) 2024-12-02T05:18:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T05:18:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:56,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:57,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116737005, max timestamp=1733116737023Num uniques 859; 2024-12-02T05:18:57,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:18:57,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116737005, max timestamp=1733116737023Num uniques 859; 2024-12-02T05:18:57,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:18:57,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116737005, max timestamp=1733116737023Num uniques 859; 2024-12-02T05:18:57,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:18:57,109 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/f8956066bb804c52b3b367fb018c2473 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f8956066bb804c52b3b367fb018c2473 2024-12-02T05:18:57,112 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/C of be075553e9fdd4e7a03804f307b7c3f9 into f8956066bb804c52b3b367fb018c2473(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:57,113 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:57,113 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/C, priority=12, startTime=1733116736228; duration=0sec 2024-12-02T05:18:57,113 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:57,113 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:C 2024-12-02T05:18:57,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733116737114, max timestamp=1733116737159Num uniques 1018; 2024-12-02T05:18:57,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-02T05:18:57,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733116737114, max timestamp=1733116737159Num uniques 1018; 2024-12-02T05:18:57,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733116737114, max timestamp=1733116737159Num uniques 1018; 2024-12-02T05:18:57,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-02T05:18:57,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-02T05:18:57,194 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/278c53564cfe43b9bfd6c1742178d85c 2024-12-02T05:18:57,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/3c9793186d6f4fedac6eed24127ce431 is 50, key is test_row_1/C:col10/1733116734986/Put/seqid=0 2024-12-02T05:18:57,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116797193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10382, totalHeapSize=23848, min timestamp=1733116737189, max timestamp=1733116737199Num uniques 1077; 2024-12-02T05:18:57,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10386, totalHeapSize=23848, min timestamp=1733116737189, max timestamp=1733116737199Num uniques 1077; 2024-12-02T05:18:57,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49312, new segment data size=10382, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-02T05:18:57,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49316, new segment data size=10386, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-02T05:18:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116797194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733116737190, max timestamp=1733116737199Num uniques 1077; 2024-12-02T05:18:57,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49312, new segment data size=6870, suffix heap size=139976,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=150 2024-12-02T05:18:57,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742270_1446 (size=9757) 2024-12-02T05:18:57,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733116737224, max timestamp=1733116737245Num uniques 1086; 2024-12-02T05:18:57,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-02T05:18:57,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733116737224, max timestamp=1733116737245Num uniques 1086; 2024-12-02T05:18:57,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-02T05:18:57,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733116737224, max timestamp=1733116737245Num uniques 1009; 2024-12-02T05:18:57,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46224, new segment data size=10806, suffix heap size=132904,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=236 2024-12-02T05:18:57,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733116737267, max timestamp=1733116737277Num uniques 1095; 2024-12-02T05:18:57,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-02T05:18:57,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733116737267, max timestamp=1733116737277Num uniques 1095; 2024-12-02T05:18:57,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733116737267, max timestamp=1733116737277Num uniques 1095; 2024-12-02T05:18:57,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-02T05:18:57,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-02T05:18:57,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116797309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116797317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116797320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116797320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116797321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116797428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116797428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116797437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116797437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116797438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,608 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/3c9793186d6f4fedac6eed24127ce431 2024-12-02T05:18:57,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/9124051fc45945ffad8d8a18040d42dd as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/9124051fc45945ffad8d8a18040d42dd 2024-12-02T05:18:57,614 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/9124051fc45945ffad8d8a18040d42dd, entries=100, sequenceid=152, filesize=9.5 K 2024-12-02T05:18:57,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/278c53564cfe43b9bfd6c1742178d85c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/278c53564cfe43b9bfd6c1742178d85c 2024-12-02T05:18:57,618 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/278c53564cfe43b9bfd6c1742178d85c, entries=100, sequenceid=152, filesize=9.5 K 2024-12-02T05:18:57,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/3c9793186d6f4fedac6eed24127ce431 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/3c9793186d6f4fedac6eed24127ce431 2024-12-02T05:18:57,621 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/3c9793186d6f4fedac6eed24127ce431, entries=100, sequenceid=152, filesize=9.5 K 2024-12-02T05:18:57,622 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for be075553e9fdd4e7a03804f307b7c3f9 in 1251ms, sequenceid=152, compaction requested=false 2024-12-02T05:18:57,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:57,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:57,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-02T05:18:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-02T05:18:57,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-02T05:18:57,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8630 sec 2024-12-02T05:18:57,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.8670 sec 2024-12-02T05:18:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:57,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=154.31 KB heapSize=432 KB 2024-12-02T05:18:57,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:18:57,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:57,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:18:57,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:57,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:18:57,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:57,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/0683aaa02b1a45a2a7648972c49d7ee5 is 50, key is test_row_0/A:col10/1733116737319/Put/seqid=0 2024-12-02T05:18:57,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742271_1447 (size=14691) 2024-12-02T05:18:57,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116797647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116797647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116797653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116797654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116797655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116797768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116797768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116797768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T05:18:57,908 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:18:57,908 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:18:57,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-02T05:18:57,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T05:18:57,910 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:18:57,910 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:18:57,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:18:57,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116797968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116797968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116797978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116797978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:57,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:57,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116797980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:57,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:57,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:58,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T05:18:58,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/0683aaa02b1a45a2a7648972c49d7ee5 2024-12-02T05:18:58,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/d217495f484e4931b6b8e2dfcf50cd66 is 50, key is test_row_0/B:col10/1733116737319/Put/seqid=0 2024-12-02T05:18:58,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742272_1448 (size=12251) 2024-12-02T05:18:58,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T05:18:58,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:58,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:58,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:58,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T05:18:58,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:58,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:58,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:58,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T05:18:58,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116798287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:58,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116798288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:58,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116798298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:58,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:58,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T05:18:58,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:58,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:58,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:58,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/d217495f484e4931b6b8e2dfcf50cd66 2024-12-02T05:18:58,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116798477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:58,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116798477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:58,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:58,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:58,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T05:18:58,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:58,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:58,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:58,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T05:18:58,678 INFO [AsyncFSWAL-0-hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData-prefix:f49461030c22,42731,1733116593060 {}] wal.AbstractFSWAL(1368): Slow sync cost: 156 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34519,DS-5a5c0e22-a56e-4f32-ae1e-a90082acf4bf,DISK]] 2024-12-02T05:18:58,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/ad676726d69f4467a6d78488df6a85ae is 50, key is test_row_0/C:col10/1733116737319/Put/seqid=0 2024-12-02T05:18:58,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742273_1449 (size=12251) 2024-12-02T05:18:58,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/ad676726d69f4467a6d78488df6a85ae 2024-12-02T05:18:58,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/0683aaa02b1a45a2a7648972c49d7ee5 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/0683aaa02b1a45a2a7648972c49d7ee5 2024-12-02T05:18:58,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/0683aaa02b1a45a2a7648972c49d7ee5, entries=200, sequenceid=261, filesize=14.3 K 2024-12-02T05:18:58,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/d217495f484e4931b6b8e2dfcf50cd66 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d217495f484e4931b6b8e2dfcf50cd66 2024-12-02T05:18:58,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d217495f484e4931b6b8e2dfcf50cd66, entries=150, sequenceid=261, filesize=12.0 K 2024-12-02T05:18:58,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/ad676726d69f4467a6d78488df6a85ae as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/ad676726d69f4467a6d78488df6a85ae 2024-12-02T05:18:58,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/ad676726d69f4467a6d78488df6a85ae, entries=150, sequenceid=261, filesize=12.0 K 2024-12-02T05:18:58,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.95 KB/442320, currentSize=26.84 KB/27480 for be075553e9fdd4e7a03804f307b7c3f9 in 1058ms, sequenceid=261, compaction requested=true 2024-12-02T05:18:58,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:58,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:18:58,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:58,701 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:58,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:18:58,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:58,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:18:58,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T05:18:58,701 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:58,704 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:58,704 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/A is initiating minor compaction (all files) 2024-12-02T05:18:58,704 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/A in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:58,704 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:58,704 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/B is initiating minor compaction (all files) 2024-12-02T05:18:58,704 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f791d6f905fb4ab7a882652aaa287f84, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/9124051fc45945ffad8d8a18040d42dd, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/0683aaa02b1a45a2a7648972c49d7ee5] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=36.0 K 2024-12-02T05:18:58,704 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/B in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:58,704 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/9cce3c1b62e44b3f85393670868af6be, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/278c53564cfe43b9bfd6c1742178d85c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d217495f484e4931b6b8e2dfcf50cd66] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=33.6 K 2024-12-02T05:18:58,705 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cce3c1b62e44b3f85393670868af6be, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733116734293 2024-12-02T05:18:58,705 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting f791d6f905fb4ab7a882652aaa287f84, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733116734293 2024-12-02T05:18:58,705 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 278c53564cfe43b9bfd6c1742178d85c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733116734986 2024-12-02T05:18:58,705 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9124051fc45945ffad8d8a18040d42dd, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733116734986 2024-12-02T05:18:58,707 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0683aaa02b1a45a2a7648972c49d7ee5, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733116737308 2024-12-02T05:18:58,707 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting d217495f484e4931b6b8e2dfcf50cd66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733116737308 2024-12-02T05:18:58,718 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#A#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:58,719 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/7e44b087d2484131a75c06f5aa898248 is 50, key is test_row_0/A:col10/1733116737319/Put/seqid=0 2024-12-02T05:18:58,720 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#B#compaction#380 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:58,721 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/b5edb82db0ad470480afc8bf9b6a6d9e is 50, key is test_row_0/B:col10/1733116737319/Put/seqid=0 2024-12-02T05:18:58,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742274_1450 (size=12593) 2024-12-02T05:18:58,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742275_1451 (size=12593) 2024-12-02T05:18:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:18:58,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T05:18:58,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:18:58,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:58,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:18:58,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:58,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:18:58,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:18:58,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/856ddb7953d6458083b54a27965419d4 is 50, key is test_row_0/A:col10/1733116737642/Put/seqid=0 2024-12-02T05:18:58,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742276_1452 (size=14741) 2024-12-02T05:18:58,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/856ddb7953d6458083b54a27965419d4 2024-12-02T05:18:58,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/ece0be0e0bf444d78fa74c36f3d31f59 is 50, key is test_row_0/B:col10/1733116737642/Put/seqid=0 2024-12-02T05:18:58,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T05:18:58,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:58,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:58,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:58,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742277_1453 (size=12301) 2024-12-02T05:18:58,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116738834, max timestamp=1733116738875Num uniques 859; 2024-12-02T05:18:58,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:18:58,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116738834, max timestamp=1733116738875Num uniques 859; 2024-12-02T05:18:58,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116738834, max timestamp=1733116738875Num uniques 859; 2024-12-02T05:18:58,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:18:58,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:18:58,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116798932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:58,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116798932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:58,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116798941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:58,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T05:18:58,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:58,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:58,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:58,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T05:18:59,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116799047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:59,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116799047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:59,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116799058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:59,126 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/7e44b087d2484131a75c06f5aa898248 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/7e44b087d2484131a75c06f5aa898248 2024-12-02T05:18:59,129 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/A of be075553e9fdd4e7a03804f307b7c3f9 into 7e44b087d2484131a75c06f5aa898248(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:59,129 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:59,129 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/A, priority=13, startTime=1733116738701; duration=0sec 2024-12-02T05:18:59,129 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:18:59,129 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:A 2024-12-02T05:18:59,130 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:18:59,130 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:18:59,130 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/C is initiating minor compaction (all files) 2024-12-02T05:18:59,130 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/C in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:59,130 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f8956066bb804c52b3b367fb018c2473, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/3c9793186d6f4fedac6eed24127ce431, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/ad676726d69f4467a6d78488df6a85ae] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=33.6 K 2024-12-02T05:18:59,131 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8956066bb804c52b3b367fb018c2473, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733116734293 2024-12-02T05:18:59,131 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c9793186d6f4fedac6eed24127ce431, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733116734986 2024-12-02T05:18:59,131 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad676726d69f4467a6d78488df6a85ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733116737308 2024-12-02T05:18:59,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T05:18:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:59,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:59,137 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#C#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:18:59,137 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/5fdea0cb02fa41e48b95fca17e312a0c is 50, key is test_row_0/C:col10/1733116737319/Put/seqid=0 2024-12-02T05:18:59,142 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/b5edb82db0ad470480afc8bf9b6a6d9e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/b5edb82db0ad470480afc8bf9b6a6d9e 2024-12-02T05:18:59,145 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/B of be075553e9fdd4e7a03804f307b7c3f9 into b5edb82db0ad470480afc8bf9b6a6d9e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:59,145 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:59,145 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/B, priority=13, startTime=1733116738701; duration=0sec 2024-12-02T05:18:59,145 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:59,145 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:B 2024-12-02T05:18:59,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742278_1454 (size=12593) 2024-12-02T05:18:59,154 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/5fdea0cb02fa41e48b95fca17e312a0c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/5fdea0cb02fa41e48b95fca17e312a0c 2024-12-02T05:18:59,158 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/C of be075553e9fdd4e7a03804f307b7c3f9 into 5fdea0cb02fa41e48b95fca17e312a0c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:18:59,158 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:59,158 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/C, priority=13, startTime=1733116738701; duration=0sec 2024-12-02T05:18:59,158 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:18:59,158 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:C 2024-12-02T05:18:59,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/ece0be0e0bf444d78fa74c36f3d31f59 2024-12-02T05:18:59,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/888556972c4646d0bab20c421608c5a5 is 50, key is test_row_0/C:col10/1733116737642/Put/seqid=0 2024-12-02T05:18:59,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742279_1455 (size=12301) 2024-12-02T05:18:59,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116799267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:59,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116799267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:59,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116799277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:59,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T05:18:59,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:59,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:59,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:59,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:59,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:59,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T05:18:59,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:59,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:59,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:59,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:59,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:59,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116799487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:59,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116799487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:59,494 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in be075553e9fdd4e7a03804f307b7c3f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:18:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,494 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_0' in be075553e9fdd4e7a03804f307b7c3f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:18:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:59,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116799577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:59,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116799577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:59,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:18:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116799587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:18:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:18:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:18:59,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T05:18:59,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:59,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:18:59,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:59,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:59,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:18:59,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/888556972c4646d0bab20c421608c5a5 2024-12-02T05:18:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/856ddb7953d6458083b54a27965419d4 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/856ddb7953d6458083b54a27965419d4 2024-12-02T05:18:59,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/856ddb7953d6458083b54a27965419d4, entries=200, sequenceid=271, filesize=14.4 K 2024-12-02T05:18:59,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/ece0be0e0bf444d78fa74c36f3d31f59 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/ece0be0e0bf444d78fa74c36f3d31f59 2024-12-02T05:18:59,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/ece0be0e0bf444d78fa74c36f3d31f59, entries=150, sequenceid=271, filesize=12.0 K 2024-12-02T05:18:59,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/888556972c4646d0bab20c421608c5a5 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/888556972c4646d0bab20c421608c5a5 2024-12-02T05:18:59,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/888556972c4646d0bab20c421608c5a5, entries=150, sequenceid=271, filesize=12.0 K 2024-12-02T05:18:59,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for be075553e9fdd4e7a03804f307b7c3f9 in 859ms, sequenceid=271, compaction requested=false 2024-12-02T05:18:59,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:18:59,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T05:18:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:18:59,749 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T05:18:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:18:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:18:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:18:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:18:59,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/59e2ef1aa0524aa2ab1f05e617b5c77f is 50, key is test_row_0/A:col10/1733116738940/Put/seqid=0 2024-12-02T05:18:59,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742280_1456 (size=12301) 2024-12-02T05:19:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T05:19:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:00,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:00,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116800105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:00,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116800114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:00,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116800115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:00,176 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/59e2ef1aa0524aa2ab1f05e617b5c77f 2024-12-02T05:19:00,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/f11e101a6f8d4ec1b40d19890b4c4e26 is 50, key is test_row_0/B:col10/1733116738940/Put/seqid=0 2024-12-02T05:19:00,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742281_1457 (size=12301) 2024-12-02T05:19:00,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116800227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:00,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:00,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116800227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:00,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116800237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:00,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:00,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116800437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116800437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:00,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116800457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:00,587 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/f11e101a6f8d4ec1b40d19890b4c4e26 2024-12-02T05:19:00,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/8a145ea10b06437bb0e6891073e8e477 is 50, key is test_row_0/C:col10/1733116738940/Put/seqid=0 2024-12-02T05:19:00,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742282_1458 (size=12301) 2024-12-02T05:19:00,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116800747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:00,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:00,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116800757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:00,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116800777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:00,997 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/8a145ea10b06437bb0e6891073e8e477 2024-12-02T05:19:01,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/59e2ef1aa0524aa2ab1f05e617b5c77f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/59e2ef1aa0524aa2ab1f05e617b5c77f 2024-12-02T05:19:01,005 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/59e2ef1aa0524aa2ab1f05e617b5c77f, entries=150, sequenceid=311, filesize=12.0 K 2024-12-02T05:19:01,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/f11e101a6f8d4ec1b40d19890b4c4e26 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/f11e101a6f8d4ec1b40d19890b4c4e26 2024-12-02T05:19:01,008 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/f11e101a6f8d4ec1b40d19890b4c4e26, entries=150, sequenceid=311, filesize=12.0 K 2024-12-02T05:19:01,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/8a145ea10b06437bb0e6891073e8e477 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/8a145ea10b06437bb0e6891073e8e477 2024-12-02T05:19:01,013 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/8a145ea10b06437bb0e6891073e8e477, entries=150, sequenceid=311, filesize=12.0 K 2024-12-02T05:19:01,013 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for be075553e9fdd4e7a03804f307b7c3f9 in 1264ms, sequenceid=311, compaction requested=true 2024-12-02T05:19:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-02T05:19:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-02T05:19:01,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-02T05:19:01,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1040 sec 2024-12-02T05:19:01,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 3.1090 sec 2024-12-02T05:19:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:01,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T05:19:01,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:19:01,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:01,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:19:01,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:01,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:19:01,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:01,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/c9da0de3084d4b3cbe4056c8c6b15f2e is 50, key is test_row_0/A:col10/1733116740104/Put/seqid=0 2024-12-02T05:19:01,278 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T05:19:01,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742283_1459 (size=14741) 2024-12-02T05:19:01,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116741302, max timestamp=1733116741332Num uniques 859; 2024-12-02T05:19:01,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116741302, max timestamp=1733116741332Num uniques 859; 2024-12-02T05:19:01,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116741302, max timestamp=1733116741332Num uniques 859; 2024-12-02T05:19:01,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:19:01,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:19:01,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:19:01,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116801375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:01,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116801375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:01,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116801376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:01,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116801488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:01,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116801497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:01,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116801497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:01,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:01,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116801507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:01,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116801507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:01,514 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in be075553e9fdd4e7a03804f307b7c3f9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:19:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:01,514 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_0' in be075553e9fdd4e7a03804f307b7c3f9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:19:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:01,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/c9da0de3084d4b3cbe4056c8c6b15f2e 2024-12-02T05:19:01,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/0e83f7d42e23484390acf4f6c834052a is 50, key is test_row_0/B:col10/1733116740104/Put/seqid=0 2024-12-02T05:19:01,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742284_1460 (size=12301) 2024-12-02T05:19:01,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116801707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:01,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:01,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116801707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:01,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:01,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:01,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116801707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:02,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116802018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:02,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116802018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:02,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116802018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:02,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T05:19:02,078 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:02,078 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-02T05:19:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T05:19:02,080 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:02,080 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:02,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:02,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/0e83f7d42e23484390acf4f6c834052a 2024-12-02T05:19:02,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/5199b7718a5e4c989a376174c616ef27 is 50, key is test_row_0/C:col10/1733116740104/Put/seqid=0 2024-12-02T05:19:02,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742285_1461 (size=12301) 2024-12-02T05:19:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T05:19:02,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-02T05:19:02,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:02,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:02,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:02,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:02,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:02,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-02T05:19:02,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:02,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:02,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:02,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:02,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T05:19:02,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/5199b7718a5e4c989a376174c616ef27 2024-12-02T05:19:02,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/c9da0de3084d4b3cbe4056c8c6b15f2e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/c9da0de3084d4b3cbe4056c8c6b15f2e 2024-12-02T05:19:02,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/c9da0de3084d4b3cbe4056c8c6b15f2e, entries=200, sequenceid=322, filesize=14.4 K 2024-12-02T05:19:02,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/0e83f7d42e23484390acf4f6c834052a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/0e83f7d42e23484390acf4f6c834052a 2024-12-02T05:19:02,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/0e83f7d42e23484390acf4f6c834052a, entries=150, sequenceid=322, filesize=12.0 K 2024-12-02T05:19:02,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/5199b7718a5e4c989a376174c616ef27 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/5199b7718a5e4c989a376174c616ef27 2024-12-02T05:19:02,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/5199b7718a5e4c989a376174c616ef27, entries=150, sequenceid=322, filesize=12.0 K 2024-12-02T05:19:02,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for be075553e9fdd4e7a03804f307b7c3f9 in 1250ms, sequenceid=322, compaction requested=true 2024-12-02T05:19:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:02,524 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:19:02,524 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:19:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:19:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:19:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:02,525 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49496 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:19:02,525 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/B is initiating minor compaction (all files) 2024-12-02T05:19:02,525 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/B in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:02,525 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/b5edb82db0ad470480afc8bf9b6a6d9e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/ece0be0e0bf444d78fa74c36f3d31f59, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/f11e101a6f8d4ec1b40d19890b4c4e26, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/0e83f7d42e23484390acf4f6c834052a] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=48.3 K 2024-12-02T05:19:02,526 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54376 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:19:02,526 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/A is initiating minor compaction (all files) 2024-12-02T05:19:02,526 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/A in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:02,526 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting b5edb82db0ad470480afc8bf9b6a6d9e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733116737308 2024-12-02T05:19:02,526 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/7e44b087d2484131a75c06f5aa898248, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/856ddb7953d6458083b54a27965419d4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/59e2ef1aa0524aa2ab1f05e617b5c77f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/c9da0de3084d4b3cbe4056c8c6b15f2e] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=53.1 K 2024-12-02T05:19:02,526 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting ece0be0e0bf444d78fa74c36f3d31f59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733116737642 2024-12-02T05:19:02,526 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e44b087d2484131a75c06f5aa898248, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733116737308 2024-12-02T05:19:02,526 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting f11e101a6f8d4ec1b40d19890b4c4e26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733116738914 2024-12-02T05:19:02,526 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 856ddb7953d6458083b54a27965419d4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733116737642 2024-12-02T05:19:02,526 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e83f7d42e23484390acf4f6c834052a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733116740104 2024-12-02T05:19:02,526 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59e2ef1aa0524aa2ab1f05e617b5c77f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733116738914 2024-12-02T05:19:02,527 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9da0de3084d4b3cbe4056c8c6b15f2e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733116740104 2024-12-02T05:19:02,533 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:02,533 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#A#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:02,534 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/34473a2f0544404ab0dd397d50b48ddf is 50, key is test_row_0/B:col10/1733116740104/Put/seqid=0 2024-12-02T05:19:02,534 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/802224bfbeed4bb2a6a0181b6ca3d3e0 is 50, key is test_row_0/A:col10/1733116740104/Put/seqid=0 2024-12-02T05:19:02,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742286_1462 (size=12779) 2024-12-02T05:19:02,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742287_1463 (size=12779) 2024-12-02T05:19:02,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-02T05:19:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:02,539 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-02T05:19:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:19:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:19:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:19:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:02,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/e976cf271e6145c582e17df17530e991 is 50, key is test_row_0/A:col10/1733116741373/Put/seqid=0 2024-12-02T05:19:02,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:02,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:02,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742288_1464 (size=12301) 2024-12-02T05:19:02,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:02,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116802558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116802570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116802570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:02,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116802677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:02,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116802677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:02,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116802677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T05:19:02,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116802897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:02,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:02,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116802897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:02,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:02,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:02,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116802897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:02,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:02,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:02,942 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/34473a2f0544404ab0dd397d50b48ddf as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/34473a2f0544404ab0dd397d50b48ddf 2024-12-02T05:19:02,942 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/802224bfbeed4bb2a6a0181b6ca3d3e0 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/802224bfbeed4bb2a6a0181b6ca3d3e0 2024-12-02T05:19:02,946 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/e976cf271e6145c582e17df17530e991 2024-12-02T05:19:02,946 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/B of be075553e9fdd4e7a03804f307b7c3f9 into 34473a2f0544404ab0dd397d50b48ddf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:02,946 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:02,946 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/A of be075553e9fdd4e7a03804f307b7c3f9 into 802224bfbeed4bb2a6a0181b6ca3d3e0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:02,946 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/B, priority=12, startTime=1733116742524; duration=0sec 2024-12-02T05:19:02,946 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:02,946 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/A, priority=12, startTime=1733116742524; duration=0sec 2024-12-02T05:19:02,946 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:02,946 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:02,946 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:A 2024-12-02T05:19:02,946 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:B 2024-12-02T05:19:02,946 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:19:02,947 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49496 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:19:02,948 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/C is initiating minor compaction (all files) 2024-12-02T05:19:02,948 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/C in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:02,948 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/5fdea0cb02fa41e48b95fca17e312a0c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/888556972c4646d0bab20c421608c5a5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/8a145ea10b06437bb0e6891073e8e477, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/5199b7718a5e4c989a376174c616ef27] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=48.3 K 2024-12-02T05:19:02,948 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fdea0cb02fa41e48b95fca17e312a0c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733116737308 2024-12-02T05:19:02,948 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 888556972c4646d0bab20c421608c5a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733116737642 2024-12-02T05:19:02,949 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a145ea10b06437bb0e6891073e8e477, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733116738914 2024-12-02T05:19:02,949 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5199b7718a5e4c989a376174c616ef27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733116740104 2024-12-02T05:19:02,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/d6d93500081b4b85ad6834ecadf32c27 is 50, key is test_row_0/B:col10/1733116741373/Put/seqid=0 2024-12-02T05:19:02,955 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#C#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:02,956 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/bedbc4b0784f47769959a50a5cb2ca97 is 50, key is test_row_0/C:col10/1733116740104/Put/seqid=0 2024-12-02T05:19:02,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742289_1465 (size=12301) 2024-12-02T05:19:02,957 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/d6d93500081b4b85ad6834ecadf32c27 2024-12-02T05:19:02,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/7bfc241b07f14738b790141737f36e38 is 50, key is test_row_0/C:col10/1733116741373/Put/seqid=0 2024-12-02T05:19:02,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742290_1466 (size=12779) 2024-12-02T05:19:02,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742291_1467 (size=12301) 2024-12-02T05:19:02,970 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/7bfc241b07f14738b790141737f36e38 2024-12-02T05:19:02,973 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/bedbc4b0784f47769959a50a5cb2ca97 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/bedbc4b0784f47769959a50a5cb2ca97 2024-12-02T05:19:02,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/e976cf271e6145c582e17df17530e991 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e976cf271e6145c582e17df17530e991 2024-12-02T05:19:02,977 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e976cf271e6145c582e17df17530e991, entries=150, sequenceid=357, filesize=12.0 K 2024-12-02T05:19:02,978 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/C of be075553e9fdd4e7a03804f307b7c3f9 into bedbc4b0784f47769959a50a5cb2ca97(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:02,978 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:02,978 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/C, priority=12, startTime=1733116742524; duration=0sec 2024-12-02T05:19:02,978 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:02,978 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:C 2024-12-02T05:19:02,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/d6d93500081b4b85ad6834ecadf32c27 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d6d93500081b4b85ad6834ecadf32c27 2024-12-02T05:19:02,981 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d6d93500081b4b85ad6834ecadf32c27, entries=150, sequenceid=357, filesize=12.0 K 2024-12-02T05:19:02,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/7bfc241b07f14738b790141737f36e38 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/7bfc241b07f14738b790141737f36e38 2024-12-02T05:19:02,984 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/7bfc241b07f14738b790141737f36e38, entries=150, sequenceid=357, filesize=12.0 K 2024-12-02T05:19:02,985 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for be075553e9fdd4e7a03804f307b7c3f9 in 446ms, sequenceid=357, compaction requested=false 2024-12-02T05:19:02,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:02,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:02,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-02T05:19:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-02T05:19:02,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-02T05:19:02,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 908 msec 2024-12-02T05:19:02,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 912 msec 2024-12-02T05:19:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:03,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T05:19:03,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:19:03,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:03,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:19:03,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:03,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:19:03,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:03,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/5a85a230529d4c53818d2cb4e4073b48 is 50, key is test_row_0/A:col10/1733116742555/Put/seqid=0 2024-12-02T05:19:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T05:19:03,217 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:03,218 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:03,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742292_1468 (size=14741) 2024-12-02T05:19:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-02T05:19:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T05:19:03,219 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:03,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/5a85a230529d4c53818d2cb4e4073b48 2024-12-02T05:19:03,220 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:03,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:03,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/c3e9ee4894774033aefeaf276839ddbe is 50, key is test_row_0/B:col10/1733116742555/Put/seqid=0 2024-12-02T05:19:03,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742293_1469 (size=12301) 2024-12-02T05:19:03,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116803259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:03,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116803260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:03,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116803261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T05:19:03,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T05:19:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:03,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:03,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:03,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116803367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:03,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:03,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116803378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116803378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:03,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T05:19:03,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:03,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:03,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:03,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:03,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T05:19:03,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116803577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:03,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:03,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116803587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:03,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:03,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116803598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:03,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:03,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/c3e9ee4894774033aefeaf276839ddbe 2024-12-02T05:19:03,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/2b5f975f672c48f3a557eeb482fd2835 is 50, key is test_row_0/C:col10/1733116742555/Put/seqid=0 2024-12-02T05:19:03,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742294_1470 (size=12301) 2024-12-02T05:19:03,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T05:19:03,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:03,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:03,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:03,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:03,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:03,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T05:19:03,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:03,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:03,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:03,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:03,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T05:19:03,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116803887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:03,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116803897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:03,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116803908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:03,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T05:19:03,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:03,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:03,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:03,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:03,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:03,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:04,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/2b5f975f672c48f3a557eeb482fd2835 2024-12-02T05:19:04,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/5a85a230529d4c53818d2cb4e4073b48 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/5a85a230529d4c53818d2cb4e4073b48 2024-12-02T05:19:04,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/5a85a230529d4c53818d2cb4e4073b48, entries=200, sequenceid=372, filesize=14.4 K 2024-12-02T05:19:04,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/c3e9ee4894774033aefeaf276839ddbe as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/c3e9ee4894774033aefeaf276839ddbe 2024-12-02T05:19:04,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/c3e9ee4894774033aefeaf276839ddbe, entries=150, sequenceid=372, filesize=12.0 K 2024-12-02T05:19:04,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/2b5f975f672c48f3a557eeb482fd2835 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2b5f975f672c48f3a557eeb482fd2835 2024-12-02T05:19:04,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2b5f975f672c48f3a557eeb482fd2835, entries=150, sequenceid=372, filesize=12.0 K 2024-12-02T05:19:04,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for be075553e9fdd4e7a03804f307b7c3f9 in 844ms, sequenceid=372, compaction requested=true 2024-12-02T05:19:04,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:04,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:04,056 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:04,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:04,056 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:04,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:19:04,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:04,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:19:04,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:04,056 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:04,056 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/A is initiating minor compaction (all files) 2024-12-02T05:19:04,056 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:04,056 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/B is initiating minor compaction (all files) 2024-12-02T05:19:04,056 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/A in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:04,056 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/B in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:04,057 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/802224bfbeed4bb2a6a0181b6ca3d3e0, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e976cf271e6145c582e17df17530e991, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/5a85a230529d4c53818d2cb4e4073b48] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=38.9 K 2024-12-02T05:19:04,057 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/34473a2f0544404ab0dd397d50b48ddf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d6d93500081b4b85ad6834ecadf32c27, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/c3e9ee4894774033aefeaf276839ddbe] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=36.5 K 2024-12-02T05:19:04,057 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 34473a2f0544404ab0dd397d50b48ddf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733116740104 2024-12-02T05:19:04,057 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 802224bfbeed4bb2a6a0181b6ca3d3e0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733116740104 2024-12-02T05:19:04,057 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting d6d93500081b4b85ad6834ecadf32c27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733116741362 2024-12-02T05:19:04,057 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting e976cf271e6145c582e17df17530e991, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733116741362 2024-12-02T05:19:04,057 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a85a230529d4c53818d2cb4e4073b48, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733116742555 2024-12-02T05:19:04,057 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e9ee4894774033aefeaf276839ddbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733116742555 2024-12-02T05:19:04,063 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#A#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:04,063 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/8caff7c0080e4b70b7c7bf0fea62c4e6 is 50, key is test_row_0/A:col10/1733116742555/Put/seqid=0 2024-12-02T05:19:04,066 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#B#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:04,066 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/6ac46af8f4fb4228b374bde3c124d461 is 50, key is test_row_0/B:col10/1733116742555/Put/seqid=0 2024-12-02T05:19:04,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742296_1472 (size=12881) 2024-12-02T05:19:04,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742295_1471 (size=12881) 2024-12-02T05:19:04,085 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/8caff7c0080e4b70b7c7bf0fea62c4e6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/8caff7c0080e4b70b7c7bf0fea62c4e6 2024-12-02T05:19:04,090 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/A of be075553e9fdd4e7a03804f307b7c3f9 into 8caff7c0080e4b70b7c7bf0fea62c4e6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:04,090 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:04,090 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/A, priority=13, startTime=1733116744055; duration=0sec 2024-12-02T05:19:04,091 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:04,091 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:A 2024-12-02T05:19:04,091 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:04,091 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:04,091 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/C is initiating minor compaction (all files) 2024-12-02T05:19:04,091 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/C in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:04,092 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/bedbc4b0784f47769959a50a5cb2ca97, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/7bfc241b07f14738b790141737f36e38, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2b5f975f672c48f3a557eeb482fd2835] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=36.5 K 2024-12-02T05:19:04,092 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting bedbc4b0784f47769959a50a5cb2ca97, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733116740104 2024-12-02T05:19:04,092 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bfc241b07f14738b790141737f36e38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733116741362 2024-12-02T05:19:04,092 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b5f975f672c48f3a557eeb482fd2835, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733116742555 2024-12-02T05:19:04,099 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#C#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:04,099 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/259f84aa106f4be1a1212246f25fc05e is 50, key is test_row_0/C:col10/1733116742555/Put/seqid=0 2024-12-02T05:19:04,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742297_1473 (size=12881) 2024-12-02T05:19:04,119 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/259f84aa106f4be1a1212246f25fc05e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/259f84aa106f4be1a1212246f25fc05e 2024-12-02T05:19:04,128 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/C of be075553e9fdd4e7a03804f307b7c3f9 into 259f84aa106f4be1a1212246f25fc05e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:04,128 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:04,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T05:19:04,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:04,137 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T05:19:04,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:19:04,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:04,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:19:04,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:04,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:19:04,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:04,139 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/C, priority=13, startTime=1733116744056; duration=0sec 2024-12-02T05:19:04,139 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:04,139 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:C 2024-12-02T05:19:04,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/36c51f6809064c75804ee1cfa6b38ae5 is 50, key is test_row_0/A:col10/1733116743260/Put/seqid=0 2024-12-02T05:19:04,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742298_1474 (size=12301) 2024-12-02T05:19:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T05:19:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:04,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:04,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116804427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:04,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116804427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:04,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:04,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116804431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:04,484 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/6ac46af8f4fb4228b374bde3c124d461 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/6ac46af8f4fb4228b374bde3c124d461 2024-12-02T05:19:04,487 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/B of be075553e9fdd4e7a03804f307b7c3f9 into 6ac46af8f4fb4228b374bde3c124d461(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:04,487 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:04,487 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/B, priority=13, startTime=1733116744056; duration=0sec 2024-12-02T05:19:04,487 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:04,487 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:B 2024-12-02T05:19:04,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116804537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:04,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116804547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:04,557 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/36c51f6809064c75804ee1cfa6b38ae5 2024-12-02T05:19:04,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/1d4641d0aa4841e4859700fbc6e4f06b is 50, key is test_row_0/B:col10/1733116743260/Put/seqid=0 2024-12-02T05:19:04,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742299_1475 (size=12301) 2024-12-02T05:19:04,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116804747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:04,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116804757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:04,972 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/1d4641d0aa4841e4859700fbc6e4f06b 2024-12-02T05:19:04,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/2f4a78747464453bae7d9c434fe99a32 is 50, key is test_row_0/C:col10/1733116743260/Put/seqid=0 2024-12-02T05:19:04,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742300_1476 (size=12301) 2024-12-02T05:19:05,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116805057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:05,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116805068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T05:19:05,382 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/2f4a78747464453bae7d9c434fe99a32 2024-12-02T05:19:05,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/36c51f6809064c75804ee1cfa6b38ae5 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/36c51f6809064c75804ee1cfa6b38ae5 2024-12-02T05:19:05,388 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/36c51f6809064c75804ee1cfa6b38ae5, entries=150, sequenceid=394, filesize=12.0 K 2024-12-02T05:19:05,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/1d4641d0aa4841e4859700fbc6e4f06b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/1d4641d0aa4841e4859700fbc6e4f06b 2024-12-02T05:19:05,391 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/1d4641d0aa4841e4859700fbc6e4f06b, entries=150, sequenceid=394, filesize=12.0 K 2024-12-02T05:19:05,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/2f4a78747464453bae7d9c434fe99a32 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2f4a78747464453bae7d9c434fe99a32 2024-12-02T05:19:05,395 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2f4a78747464453bae7d9c434fe99a32, entries=150, sequenceid=394, filesize=12.0 K 2024-12-02T05:19:05,396 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for be075553e9fdd4e7a03804f307b7c3f9 in 1259ms, sequenceid=394, compaction requested=false 2024-12-02T05:19:05,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:05,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:05,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-02T05:19:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-02T05:19:05,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-02T05:19:05,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1760 sec 2024-12-02T05:19:05,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.1800 sec 2024-12-02T05:19:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:05,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T05:19:05,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:19:05,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:05,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:19:05,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:05,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:19:05,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:05,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/2e7d195fc5b14d6694af6b836c31d0a8 is 50, key is test_row_0/A:col10/1733116745443/Put/seqid=0 2024-12-02T05:19:05,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742301_1477 (size=12301) 2024-12-02T05:19:05,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41504 deadline: 1733116805547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:05,558 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in be075553e9fdd4e7a03804f307b7c3f9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:19:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:05,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41466 deadline: 1733116805558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:05,565 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_0' in be075553e9fdd4e7a03804f307b7c3f9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:19:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:05,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116805560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:05,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116805568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:05,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116805577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:05,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116805677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:05,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/2e7d195fc5b14d6694af6b836c31d0a8 2024-12-02T05:19:05,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/94a6eb03e19d41d99b695d86570b3760 is 50, key is test_row_0/B:col10/1733116745443/Put/seqid=0 2024-12-02T05:19:05,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742302_1478 (size=12301) 2024-12-02T05:19:05,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/94a6eb03e19d41d99b695d86570b3760 2024-12-02T05:19:05,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/4c7ebf759df9478e9e94667b3b55aff1 is 50, key is test_row_0/C:col10/1733116745443/Put/seqid=0 2024-12-02T05:19:05,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742303_1479 (size=12301) 2024-12-02T05:19:05,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116805887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:06,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116806198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:06,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:06,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/4c7ebf759df9478e9e94667b3b55aff1 2024-12-02T05:19:06,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/2e7d195fc5b14d6694af6b836c31d0a8 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/2e7d195fc5b14d6694af6b836c31d0a8 2024-12-02T05:19:06,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/2e7d195fc5b14d6694af6b836c31d0a8, entries=150, sequenceid=408, filesize=12.0 K 2024-12-02T05:19:06,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/94a6eb03e19d41d99b695d86570b3760 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/94a6eb03e19d41d99b695d86570b3760 2024-12-02T05:19:06,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/94a6eb03e19d41d99b695d86570b3760, entries=150, sequenceid=408, filesize=12.0 K 2024-12-02T05:19:06,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/4c7ebf759df9478e9e94667b3b55aff1 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/4c7ebf759df9478e9e94667b3b55aff1 2024-12-02T05:19:06,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/4c7ebf759df9478e9e94667b3b55aff1, entries=150, sequenceid=408, filesize=12.0 K 2024-12-02T05:19:06,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for be075553e9fdd4e7a03804f307b7c3f9 in 848ms, sequenceid=408, compaction requested=true 2024-12-02T05:19:06,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:06,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:06,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:06,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:19:06,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:06,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:19:06,293 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:06,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:06,293 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:06,293 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:06,293 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:06,293 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/A is initiating minor compaction (all files) 2024-12-02T05:19:06,293 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/B is initiating minor compaction (all files) 2024-12-02T05:19:06,293 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/B in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:06,293 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/A in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:06,293 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/8caff7c0080e4b70b7c7bf0fea62c4e6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/36c51f6809064c75804ee1cfa6b38ae5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/2e7d195fc5b14d6694af6b836c31d0a8] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=36.6 K 2024-12-02T05:19:06,293 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/6ac46af8f4fb4228b374bde3c124d461, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/1d4641d0aa4841e4859700fbc6e4f06b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/94a6eb03e19d41d99b695d86570b3760] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=36.6 K 2024-12-02T05:19:06,294 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ac46af8f4fb4228b374bde3c124d461, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733116742555 2024-12-02T05:19:06,294 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8caff7c0080e4b70b7c7bf0fea62c4e6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733116742555 2024-12-02T05:19:06,294 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d4641d0aa4841e4859700fbc6e4f06b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733116743246 2024-12-02T05:19:06,294 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36c51f6809064c75804ee1cfa6b38ae5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733116743246 2024-12-02T05:19:06,294 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 94a6eb03e19d41d99b695d86570b3760, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733116744426 2024-12-02T05:19:06,294 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e7d195fc5b14d6694af6b836c31d0a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733116744426 2024-12-02T05:19:06,299 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#A#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:06,300 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/8f0001e2ba184c7eab75f3922f50fa1c is 50, key is test_row_0/A:col10/1733116745443/Put/seqid=0 2024-12-02T05:19:06,306 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#B#compaction#411 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:06,306 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/c2b185149aa34f5e9df90b8d798021ae is 50, key is test_row_0/B:col10/1733116745443/Put/seqid=0 2024-12-02T05:19:06,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742304_1480 (size=12983) 2024-12-02T05:19:06,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742305_1481 (size=12983) 2024-12-02T05:19:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:06,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T05:19:06,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:19:06,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:06,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:19:06,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:06,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:19:06,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:06,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/0563f99434334ed78b5430ec0e13bf58 is 50, key is test_row_0/A:col10/1733116745556/Put/seqid=0 2024-12-02T05:19:06,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742306_1482 (size=14741) 2024-12-02T05:19:06,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/0563f99434334ed78b5430ec0e13bf58 2024-12-02T05:19:06,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/4f3b4d2a8b574774880a0c03876b1461 is 50, key is test_row_0/B:col10/1733116745556/Put/seqid=0 2024-12-02T05:19:06,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742307_1483 (size=12301) 2024-12-02T05:19:06,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/4f3b4d2a8b574774880a0c03876b1461 2024-12-02T05:19:06,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/e0216515387a4f3dbc7921a91a7957d8 is 50, key is test_row_0/C:col10/1733116745556/Put/seqid=0 2024-12-02T05:19:06,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116806629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:06,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116806637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:06,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742308_1484 (size=12301) 2024-12-02T05:19:06,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/e0216515387a4f3dbc7921a91a7957d8 2024-12-02T05:19:06,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/0563f99434334ed78b5430ec0e13bf58 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/0563f99434334ed78b5430ec0e13bf58 2024-12-02T05:19:06,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/0563f99434334ed78b5430ec0e13bf58, entries=200, sequenceid=428, filesize=14.4 K 2024-12-02T05:19:06,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/4f3b4d2a8b574774880a0c03876b1461 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/4f3b4d2a8b574774880a0c03876b1461 2024-12-02T05:19:06,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/4f3b4d2a8b574774880a0c03876b1461, entries=150, sequenceid=428, filesize=12.0 K 2024-12-02T05:19:06,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/e0216515387a4f3dbc7921a91a7957d8 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e0216515387a4f3dbc7921a91a7957d8 2024-12-02T05:19:06,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e0216515387a4f3dbc7921a91a7957d8, entries=150, sequenceid=428, filesize=12.0 K 2024-12-02T05:19:06,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for be075553e9fdd4e7a03804f307b7c3f9 in 67ms, sequenceid=428, compaction requested=true 2024-12-02T05:19:06,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:06,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:A, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:19:06,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T05:19:06,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:19:06,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T05:19:06,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:19:06,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T05:19:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:06,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T05:19:06,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:19:06,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:06,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:19:06,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:06,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:19:06,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:06,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/650bbf9d1ccb4143aa9fbab65b786401 is 50, key is test_row_0/A:col10/1733116746708/Put/seqid=0 2024-12-02T05:19:06,713 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/8f0001e2ba184c7eab75f3922f50fa1c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/8f0001e2ba184c7eab75f3922f50fa1c 2024-12-02T05:19:06,719 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/c2b185149aa34f5e9df90b8d798021ae as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/c2b185149aa34f5e9df90b8d798021ae 2024-12-02T05:19:06,719 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/A of be075553e9fdd4e7a03804f307b7c3f9 into 8f0001e2ba184c7eab75f3922f50fa1c(size=12.7 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:06,719 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:06,719 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/A, priority=13, startTime=1733116746292; duration=0sec 2024-12-02T05:19:06,719 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T05:19:06,719 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:A 2024-12-02T05:19:06,720 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:A 2024-12-02T05:19:06,720 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:19:06,720 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:19:06,720 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/C is initiating minor compaction (all files) 2024-12-02T05:19:06,720 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/C in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:06,721 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/259f84aa106f4be1a1212246f25fc05e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2f4a78747464453bae7d9c434fe99a32, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/4c7ebf759df9478e9e94667b3b55aff1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e0216515387a4f3dbc7921a91a7957d8] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=48.6 K 2024-12-02T05:19:06,721 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 259f84aa106f4be1a1212246f25fc05e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733116742555 2024-12-02T05:19:06,721 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f4a78747464453bae7d9c434fe99a32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733116743246 2024-12-02T05:19:06,722 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c7ebf759df9478e9e94667b3b55aff1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733116744426 2024-12-02T05:19:06,722 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0216515387a4f3dbc7921a91a7957d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733116745537 2024-12-02T05:19:06,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742309_1485 (size=14741) 2024-12-02T05:19:06,725 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/B of be075553e9fdd4e7a03804f307b7c3f9 into c2b185149aa34f5e9df90b8d798021ae(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:06,725 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:06,725 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/B, priority=13, startTime=1733116746293; duration=0sec 2024-12-02T05:19:06,725 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T05:19:06,725 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:B 2024-12-02T05:19:06,725 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:B 2024-12-02T05:19:06,725 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-12-02T05:19:06,725 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T05:19:06,725 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T05:19:06,725 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. because compaction request was cancelled 2024-12-02T05:19:06,725 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:C 2024-12-02T05:19:06,725 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T05:19:06,726 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T05:19:06,726 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T05:19:06,726 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. because compaction request was cancelled 2024-12-02T05:19:06,726 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:B 2024-12-02T05:19:06,726 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T05:19:06,727 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T05:19:06,727 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T05:19:06,727 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. because compaction request was cancelled 2024-12-02T05:19:06,727 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:A 2024-12-02T05:19:06,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/650bbf9d1ccb4143aa9fbab65b786401 2024-12-02T05:19:06,731 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#C#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:06,731 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/f621d1c9c7bf4a2dabf7e70c102d1d64 is 50, key is test_row_0/C:col10/1733116745556/Put/seqid=0 2024-12-02T05:19:06,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/4147fb0c339742418617f1b2407abf99 is 50, key is test_row_0/B:col10/1733116746708/Put/seqid=0 2024-12-02T05:19:06,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742310_1486 (size=13017) 2024-12-02T05:19:06,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742311_1487 (size=12301) 2024-12-02T05:19:06,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116806772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:06,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116806772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116806774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:06,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116806888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:06,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116806888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116806888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:07,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116807098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116807098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116807098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:07,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/4147fb0c339742418617f1b2407abf99 2024-12-02T05:19:07,140 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/f621d1c9c7bf4a2dabf7e70c102d1d64 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f621d1c9c7bf4a2dabf7e70c102d1d64 2024-12-02T05:19:07,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/2cf20242589941319a22b408fc1132b0 is 50, key is test_row_0/C:col10/1733116746708/Put/seqid=0 2024-12-02T05:19:07,149 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/C of be075553e9fdd4e7a03804f307b7c3f9 into f621d1c9c7bf4a2dabf7e70c102d1d64(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:07,149 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:07,149 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/C, priority=12, startTime=1733116746660; duration=0sec 2024-12-02T05:19:07,149 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:07,149 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:C 2024-12-02T05:19:07,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742312_1488 (size=12301) 2024-12-02T05:19:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T05:19:07,387 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:07,388 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-02T05:19:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T05:19:07,389 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:07,390 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:07,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:07,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116807417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116807418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:07,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116807418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:07,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T05:19:07,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-02T05:19:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:07,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:07,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:07,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/2cf20242589941319a22b408fc1132b0 2024-12-02T05:19:07,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/650bbf9d1ccb4143aa9fbab65b786401 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/650bbf9d1ccb4143aa9fbab65b786401 2024-12-02T05:19:07,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/650bbf9d1ccb4143aa9fbab65b786401, entries=200, sequenceid=441, filesize=14.4 K 2024-12-02T05:19:07,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/4147fb0c339742418617f1b2407abf99 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/4147fb0c339742418617f1b2407abf99 2024-12-02T05:19:07,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/4147fb0c339742418617f1b2407abf99, entries=150, sequenceid=441, filesize=12.0 K 2024-12-02T05:19:07,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/2cf20242589941319a22b408fc1132b0 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2cf20242589941319a22b408fc1132b0 2024-12-02T05:19:07,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2cf20242589941319a22b408fc1132b0, entries=150, sequenceid=441, filesize=12.0 K 2024-12-02T05:19:07,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for be075553e9fdd4e7a03804f307b7c3f9 in 860ms, sequenceid=441, compaction requested=true 2024-12-02T05:19:07,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:07,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:07,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:07,569 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:07,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:19:07,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:07,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:19:07,569 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:07,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:07,570 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:07,570 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:07,570 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/A is initiating minor compaction (all files) 2024-12-02T05:19:07,570 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/B is initiating minor compaction (all files) 2024-12-02T05:19:07,570 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/A in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:07,570 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/B in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:07,570 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/8f0001e2ba184c7eab75f3922f50fa1c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/0563f99434334ed78b5430ec0e13bf58, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/650bbf9d1ccb4143aa9fbab65b786401] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=41.5 K 2024-12-02T05:19:07,570 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/c2b185149aa34f5e9df90b8d798021ae, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/4f3b4d2a8b574774880a0c03876b1461, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/4147fb0c339742418617f1b2407abf99] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=36.7 K 2024-12-02T05:19:07,570 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f0001e2ba184c7eab75f3922f50fa1c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733116744426 2024-12-02T05:19:07,570 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting c2b185149aa34f5e9df90b8d798021ae, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733116744426 2024-12-02T05:19:07,571 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0563f99434334ed78b5430ec0e13bf58, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733116745520 2024-12-02T05:19:07,571 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f3b4d2a8b574774880a0c03876b1461, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733116745537 2024-12-02T05:19:07,571 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 650bbf9d1ccb4143aa9fbab65b786401, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733116746610 2024-12-02T05:19:07,571 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 4147fb0c339742418617f1b2407abf99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733116746625 2024-12-02T05:19:07,585 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:07,585 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#A#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:07,585 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/89f5cc23f08b4223b9a3109de2afac21 is 50, key is test_row_0/B:col10/1733116746708/Put/seqid=0 2024-12-02T05:19:07,585 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/f0e08f0a18ba4cbd929a0f6aa5c4fc56 is 50, key is test_row_0/A:col10/1733116746708/Put/seqid=0 2024-12-02T05:19:07,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742313_1489 (size=13085) 2024-12-02T05:19:07,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742314_1490 (size=13085) 2024-12-02T05:19:07,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-02T05:19:07,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:07,695 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T05:19:07,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:19:07,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:07,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:19:07,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:07,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:19:07,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:07,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/cbecd4d09e55411583942de50d039d84 is 50, key is test_row_0/A:col10/1733116746771/Put/seqid=0 2024-12-02T05:19:07,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T05:19:07,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742315_1491 (size=12301) 2024-12-02T05:19:07,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:07,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:07,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116807952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:07,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116807957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:07,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116807957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:08,014 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/89f5cc23f08b4223b9a3109de2afac21 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/89f5cc23f08b4223b9a3109de2afac21 2024-12-02T05:19:08,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T05:19:08,020 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/f0e08f0a18ba4cbd929a0f6aa5c4fc56 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f0e08f0a18ba4cbd929a0f6aa5c4fc56 2024-12-02T05:19:08,021 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/B of be075553e9fdd4e7a03804f307b7c3f9 into 89f5cc23f08b4223b9a3109de2afac21(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:08,021 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:08,021 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/B, priority=13, startTime=1733116747569; duration=0sec 2024-12-02T05:19:08,022 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:08,022 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:B 2024-12-02T05:19:08,022 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T05:19:08,022 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T05:19:08,022 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T05:19:08,022 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. because compaction request was cancelled 2024-12-02T05:19:08,022 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:C 2024-12-02T05:19:08,025 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/A of be075553e9fdd4e7a03804f307b7c3f9 into f0e08f0a18ba4cbd929a0f6aa5c4fc56(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:08,025 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:08,025 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/A, priority=13, startTime=1733116747569; duration=0sec 2024-12-02T05:19:08,025 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:08,025 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:A 2024-12-02T05:19:08,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116808068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:08,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116808077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:08,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116808078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:08,109 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/cbecd4d09e55411583942de50d039d84 2024-12-02T05:19:08,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/3e0708d4ae9248bd8ac2d2dd940ad24f is 50, key is test_row_0/B:col10/1733116746771/Put/seqid=0 2024-12-02T05:19:08,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742316_1492 (size=12301) 2024-12-02T05:19:08,160 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/3e0708d4ae9248bd8ac2d2dd940ad24f 2024-12-02T05:19:08,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/0c39a9d7488d427e84ba877a0a84de65 is 50, key is test_row_0/C:col10/1733116746771/Put/seqid=0 2024-12-02T05:19:08,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742317_1493 (size=12301) 2024-12-02T05:19:08,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116808277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:08,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116808287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:08,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116808287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T05:19:08,573 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/0c39a9d7488d427e84ba877a0a84de65 2024-12-02T05:19:08,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:08,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116808588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:08,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:08,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116808597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116808597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:08,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/cbecd4d09e55411583942de50d039d84 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/cbecd4d09e55411583942de50d039d84 2024-12-02T05:19:08,854 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/cbecd4d09e55411583942de50d039d84, entries=150, sequenceid=463, filesize=12.0 K 2024-12-02T05:19:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/3e0708d4ae9248bd8ac2d2dd940ad24f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/3e0708d4ae9248bd8ac2d2dd940ad24f 2024-12-02T05:19:08,859 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/3e0708d4ae9248bd8ac2d2dd940ad24f, entries=150, sequenceid=463, filesize=12.0 K 2024-12-02T05:19:08,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/0c39a9d7488d427e84ba877a0a84de65 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/0c39a9d7488d427e84ba877a0a84de65 2024-12-02T05:19:08,871 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/0c39a9d7488d427e84ba877a0a84de65, entries=150, sequenceid=463, filesize=12.0 K 2024-12-02T05:19:08,872 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for be075553e9fdd4e7a03804f307b7c3f9 in 1177ms, sequenceid=463, compaction requested=true 2024-12-02T05:19:08,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:08,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:08,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-02T05:19:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-02T05:19:08,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-02T05:19:08,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4830 sec 2024-12-02T05:19:08,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.4870 sec 2024-12-02T05:19:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:09,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T05:19:09,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:19:09,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:09,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:19:09,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:09,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:19:09,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:09,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/59e97c7f71dd40db93f5bfd3e28684fa is 50, key is test_row_0/A:col10/1733116747955/Put/seqid=0 2024-12-02T05:19:09,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742318_1494 (size=14741) 2024-12-02T05:19:09,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116809144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:09,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116809145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:09,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116809146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:09,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116809257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:09,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:09,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116809267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116809268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:09,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116809467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:09,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116809477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:09,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116809477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:09,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/59e97c7f71dd40db93f5bfd3e28684fa 2024-12-02T05:19:09,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/fbd970fdee7d48a3a34bc1daec1fef1e is 50, key is test_row_0/B:col10/1733116747955/Put/seqid=0 2024-12-02T05:19:09,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742319_1495 (size=12301) 2024-12-02T05:19:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T05:19:09,537 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:09,538 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-02T05:19:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T05:19:09,539 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:09,539 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:09,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T05:19:09,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T05:19:09,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:09,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:09,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:09,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:09,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:09,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116809777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:09,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116809787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:09,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116809787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:09,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T05:19:09,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:09,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:09,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:09,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:09,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T05:19:09,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/fbd970fdee7d48a3a34bc1daec1fef1e 2024-12-02T05:19:09,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/737966cf61994682b3d3107584c60df7 is 50, key is test_row_0/C:col10/1733116747955/Put/seqid=0 2024-12-02T05:19:09,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742320_1496 (size=12301) 2024-12-02T05:19:09,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T05:19:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:09,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:09,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:10,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T05:19:10,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:10,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:10,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:10,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:10,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T05:19:10,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1733116810287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:10,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T05:19:10,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:10,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:10,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:10,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:10,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:10,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1733116810297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:10,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41488 deadline: 1733116810297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 , the old value is region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., hostname=f49461030c22,46495,1733116594459, seqNum=2 because the exception is null or not the one we care about 2024-12-02T05:19:10,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/737966cf61994682b3d3107584c60df7 2024-12-02T05:19:10,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/59e97c7f71dd40db93f5bfd3e28684fa as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/59e97c7f71dd40db93f5bfd3e28684fa 2024-12-02T05:19:10,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/59e97c7f71dd40db93f5bfd3e28684fa, entries=200, sequenceid=480, filesize=14.4 K 2024-12-02T05:19:10,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/fbd970fdee7d48a3a34bc1daec1fef1e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/fbd970fdee7d48a3a34bc1daec1fef1e 2024-12-02T05:19:10,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/fbd970fdee7d48a3a34bc1daec1fef1e, entries=150, sequenceid=480, filesize=12.0 K 2024-12-02T05:19:10,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/737966cf61994682b3d3107584c60df7 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/737966cf61994682b3d3107584c60df7 2024-12-02T05:19:10,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/737966cf61994682b3d3107584c60df7, entries=150, sequenceid=480, filesize=12.0 K 2024-12-02T05:19:10,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for be075553e9fdd4e7a03804f307b7c3f9 in 1244ms, sequenceid=480, compaction requested=true 2024-12-02T05:19:10,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:10,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:10,355 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:10,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:10,355 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:10,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:19:10,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:10,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:19:10,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:10,356 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:10,357 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/A is initiating minor compaction (all files) 2024-12-02T05:19:10,357 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/A in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:10,357 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f0e08f0a18ba4cbd929a0f6aa5c4fc56, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/cbecd4d09e55411583942de50d039d84, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/59e97c7f71dd40db93f5bfd3e28684fa] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=39.2 K 2024-12-02T05:19:10,357 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:10,357 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/B is initiating minor compaction (all files) 2024-12-02T05:19:10,357 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/B in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:10,357 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/89f5cc23f08b4223b9a3109de2afac21, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/3e0708d4ae9248bd8ac2d2dd940ad24f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/fbd970fdee7d48a3a34bc1daec1fef1e] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=36.8 K 2024-12-02T05:19:10,357 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0e08f0a18ba4cbd929a0f6aa5c4fc56, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733116746625 2024-12-02T05:19:10,358 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 89f5cc23f08b4223b9a3109de2afac21, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733116746625 2024-12-02T05:19:10,358 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbecd4d09e55411583942de50d039d84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733116746770 2024-12-02T05:19:10,358 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59e97c7f71dd40db93f5bfd3e28684fa, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1733116747942 2024-12-02T05:19:10,358 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e0708d4ae9248bd8ac2d2dd940ad24f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733116746770 2024-12-02T05:19:10,358 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting fbd970fdee7d48a3a34bc1daec1fef1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1733116747942 2024-12-02T05:19:10,367 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#B#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:10,368 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/3616ef64f5084326a2dce64015eb90cc is 50, key is test_row_0/B:col10/1733116747955/Put/seqid=0 2024-12-02T05:19:10,373 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#A#compaction#428 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:10,374 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/d17df7f68e3848feaab14d0a3c3d311b is 50, key is test_row_0/A:col10/1733116747955/Put/seqid=0 2024-12-02T05:19:10,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742321_1497 (size=13187) 2024-12-02T05:19:10,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742322_1498 (size=13187) 2024-12-02T05:19:10,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T05:19:10,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:10,465 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T05:19:10,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:19:10,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:10,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:19:10,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:10,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:19:10,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:10,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/7aac5eb51fed4be29513834d60dcedc8 is 50, key is test_row_0/A:col10/1733116749145/Put/seqid=0 2024-12-02T05:19:10,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742323_1499 (size=12301) 2024-12-02T05:19:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T05:19:10,787 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/3616ef64f5084326a2dce64015eb90cc as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/3616ef64f5084326a2dce64015eb90cc 2024-12-02T05:19:10,791 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/B of be075553e9fdd4e7a03804f307b7c3f9 into 3616ef64f5084326a2dce64015eb90cc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:10,792 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:10,792 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/B, priority=13, startTime=1733116750355; duration=0sec 2024-12-02T05:19:10,792 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:10,792 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:B 2024-12-02T05:19:10,792 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:19:10,793 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:19:10,793 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): be075553e9fdd4e7a03804f307b7c3f9/C is initiating minor compaction (all files) 2024-12-02T05:19:10,793 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be075553e9fdd4e7a03804f307b7c3f9/C in TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:10,793 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f621d1c9c7bf4a2dabf7e70c102d1d64, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2cf20242589941319a22b408fc1132b0, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/0c39a9d7488d427e84ba877a0a84de65, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/737966cf61994682b3d3107584c60df7] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp, totalSize=48.8 K 2024-12-02T05:19:10,793 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting f621d1c9c7bf4a2dabf7e70c102d1d64, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733116745537 2024-12-02T05:19:10,794 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cf20242589941319a22b408fc1132b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733116746625 2024-12-02T05:19:10,794 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c39a9d7488d427e84ba877a0a84de65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733116746770 2024-12-02T05:19:10,794 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 737966cf61994682b3d3107584c60df7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1733116747942 2024-12-02T05:19:10,796 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/d17df7f68e3848feaab14d0a3c3d311b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/d17df7f68e3848feaab14d0a3c3d311b 2024-12-02T05:19:10,800 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/A of be075553e9fdd4e7a03804f307b7c3f9 into d17df7f68e3848feaab14d0a3c3d311b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:10,800 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:10,800 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/A, priority=13, startTime=1733116750355; duration=0sec 2024-12-02T05:19:10,800 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:10,800 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:A 2024-12-02T05:19:10,801 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be075553e9fdd4e7a03804f307b7c3f9#C#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:10,802 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/5a36f72a46734bbd9abc87207dbf383f is 50, key is test_row_0/C:col10/1733116747955/Put/seqid=0 2024-12-02T05:19:10,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742324_1500 (size=13153) 2024-12-02T05:19:10,809 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/5a36f72a46734bbd9abc87207dbf383f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/5a36f72a46734bbd9abc87207dbf383f 2024-12-02T05:19:10,813 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be075553e9fdd4e7a03804f307b7c3f9/C of be075553e9fdd4e7a03804f307b7c3f9 into 5a36f72a46734bbd9abc87207dbf383f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:10,813 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:10,813 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9., storeName=be075553e9fdd4e7a03804f307b7c3f9/C, priority=12, startTime=1733116750356; duration=0sec 2024-12-02T05:19:10,813 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:10,813 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:C 2024-12-02T05:19:10,879 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/7aac5eb51fed4be29513834d60dcedc8 2024-12-02T05:19:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/07acc76a8eb74cdbbcb48a81bca86566 is 50, key is test_row_0/B:col10/1733116749145/Put/seqid=0 2024-12-02T05:19:10,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742325_1501 (size=12301) 2024-12-02T05:19:11,301 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/07acc76a8eb74cdbbcb48a81bca86566 2024-12-02T05:19:11,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. as already flushing 2024-12-02T05:19:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:11,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/ad405d37e1da4baf9bdd2edd36362bc7 is 50, key is test_row_0/C:col10/1733116749145/Put/seqid=0 2024-12-02T05:19:11,310 INFO [Thread-1910 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1910. 2024-12-02T05:19:11,310 DEBUG [Thread-1910 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:11,310 DEBUG [Thread-1910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,310 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:11,311 DEBUG [Thread-1910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,312 INFO [Thread-1905 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1905. 2024-12-02T05:19:11,312 DEBUG [Thread-1905 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:11,312 DEBUG [Thread-1905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,312 INFO [Thread-1912 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1912. 2024-12-02T05:19:11,312 DEBUG [Thread-1912 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:11,312 DEBUG [Thread-1912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,312 DEBUG [Thread-1905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:11,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742326_1502 (size=12301) 2024-12-02T05:19:11,312 DEBUG [Thread-1912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:11,313 INFO [Thread-1914 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1914. 2024-12-02T05:19:11,313 INFO [Thread-1916 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1916. 2024-12-02T05:19:11,313 DEBUG [Thread-1914 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:11,313 DEBUG [Thread-1916 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:11,313 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,313 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,313 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,313 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,313 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:11,313 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:11,314 INFO [Thread-1903 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1903. 2024-12-02T05:19:11,314 DEBUG [Thread-1903 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:11,314 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,314 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:11,314 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/ad405d37e1da4baf9bdd2edd36362bc7 2024-12-02T05:19:11,315 INFO [Thread-1918 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1918. 2024-12-02T05:19:11,315 DEBUG [Thread-1918 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:11,315 DEBUG [Thread-1918 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,315 DEBUG [Thread-1918 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:11,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/7aac5eb51fed4be29513834d60dcedc8 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/7aac5eb51fed4be29513834d60dcedc8 2024-12-02T05:19:11,319 INFO [Thread-1899 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1899. 2024-12-02T05:19:11,319 DEBUG [Thread-1899 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:11,319 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,319 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:11,319 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:11,321 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/7aac5eb51fed4be29513834d60dcedc8, entries=150, sequenceid=497, filesize=12.0 K 2024-12-02T05:19:11,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/07acc76a8eb74cdbbcb48a81bca86566 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/07acc76a8eb74cdbbcb48a81bca86566 2024-12-02T05:19:11,324 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/07acc76a8eb74cdbbcb48a81bca86566, entries=150, sequenceid=497, filesize=12.0 K 2024-12-02T05:19:11,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/ad405d37e1da4baf9bdd2edd36362bc7 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/ad405d37e1da4baf9bdd2edd36362bc7 2024-12-02T05:19:11,327 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/ad405d37e1da4baf9bdd2edd36362bc7, entries=150, sequenceid=497, filesize=12.0 K 2024-12-02T05:19:11,327 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=33.54 KB/34350 for be075553e9fdd4e7a03804f307b7c3f9 in 862ms, sequenceid=497, compaction requested=false 2024-12-02T05:19:11,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:11,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:11,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-02T05:19:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-02T05:19:11,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-02T05:19:11,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7890 sec 2024-12-02T05:19:11,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.7920 sec 2024-12-02T05:19:11,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T05:19:11,688 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:15,621 INFO [Thread-1907 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1907. 2024-12-02T05:19:15,621 DEBUG [Thread-1907 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:15,621 DEBUG [Thread-1907 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:15,621 DEBUG [Thread-1907 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:15,621 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:15,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be075553e9fdd4e7a03804f307b7c3f9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T05:19:15,640 INFO [Thread-1901 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1901. 2024-12-02T05:19:15,640 DEBUG [Thread-1901 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:15,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=A 2024-12-02T05:19:15,640 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:15,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:15,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=B 2024-12-02T05:19:15,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:15,640 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:15,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be075553e9fdd4e7a03804f307b7c3f9, store=C 2024-12-02T05:19:15,640 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:15,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 134 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1521 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4563 rows 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1530 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4583 rows 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1530 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4590 rows 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1535 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4599 rows 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1526 2024-12-02T05:19:15,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4577 rows 2024-12-02T05:19:15,642 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T05:19:15,642 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T05:19:15,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:15,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:15,642 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:15,643 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T05:19:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:15,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/bcd9e9d1f8eb47ce90d3ef4ab60b9bac is 50, key is test_row_0/A:col10/1733116755638/Put/seqid=0 2024-12-02T05:19:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T05:19:15,648 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116755648"}]},"ts":"1733116755648"} 2024-12-02T05:19:15,649 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T05:19:15,650 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T05:19:15,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742327_1503 (size=12301) 2024-12-02T05:19:15,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T05:19:15,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be075553e9fdd4e7a03804f307b7c3f9, UNASSIGN}] 2024-12-02T05:19:15,653 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be075553e9fdd4e7a03804f307b7c3f9, UNASSIGN 2024-12-02T05:19:15,653 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=be075553e9fdd4e7a03804f307b7c3f9, regionState=CLOSING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:19:15,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=be075553e9fdd4e7a03804f307b7c3f9, UNASSIGN because future has completed 2024-12-02T05:19:15,655 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T05:19:15,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:19:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T05:19:15,807 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:15,807 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T05:19:15,807 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing be075553e9fdd4e7a03804f307b7c3f9, disabling compactions & flushes 2024-12-02T05:19:15,807 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T05:19:16,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/bcd9e9d1f8eb47ce90d3ef4ab60b9bac 2024-12-02T05:19:16,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/338c47471d2c43cd9feac9753b4618a1 is 50, key is test_row_0/B:col10/1733116755638/Put/seqid=0 2024-12-02T05:19:16,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742328_1504 (size=12301) 2024-12-02T05:19:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T05:19:16,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/338c47471d2c43cd9feac9753b4618a1 2024-12-02T05:19:16,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/ad2a290371a04d71abffd0ffb01d113b is 50, key is test_row_0/C:col10/1733116755638/Put/seqid=0 2024-12-02T05:19:16,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742329_1505 (size=12301) 2024-12-02T05:19:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T05:19:16,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/ad2a290371a04d71abffd0ffb01d113b 2024-12-02T05:19:16,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/A/bcd9e9d1f8eb47ce90d3ef4ab60b9bac as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/bcd9e9d1f8eb47ce90d3ef4ab60b9bac 2024-12-02T05:19:16,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/bcd9e9d1f8eb47ce90d3ef4ab60b9bac, entries=150, sequenceid=510, filesize=12.0 K 2024-12-02T05:19:16,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/B/338c47471d2c43cd9feac9753b4618a1 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/338c47471d2c43cd9feac9753b4618a1 2024-12-02T05:19:16,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/338c47471d2c43cd9feac9753b4618a1, entries=150, sequenceid=510, filesize=12.0 K 2024-12-02T05:19:16,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/.tmp/C/ad2a290371a04d71abffd0ffb01d113b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/ad2a290371a04d71abffd0ffb01d113b 2024-12-02T05:19:16,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/ad2a290371a04d71abffd0ffb01d113b, entries=150, sequenceid=510, filesize=12.0 K 2024-12-02T05:19:16,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for be075553e9fdd4e7a03804f307b7c3f9 in 1263ms, sequenceid=510, compaction requested=true 2024-12-02T05:19:16,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be075553e9fdd4e7a03804f307b7c3f9: 2024-12-02T05:19:16,903 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:16,903 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:16,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:16,903 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. because compaction request was cancelled 2024-12-02T05:19:16,903 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. after waiting 0 ms 2024-12-02T05:19:16,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:16,903 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:16,903 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:A 2024-12-02T05:19:16,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:19:16,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:16,903 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. because compaction request was cancelled 2024-12-02T05:19:16,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be075553e9fdd4e7a03804f307b7c3f9:C, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:19:16,903 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:B 2024-12-02T05:19:16,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:16,903 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. because compaction request was cancelled 2024-12-02T05:19:16,903 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be075553e9fdd4e7a03804f307b7c3f9:C 2024-12-02T05:19:16,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/04a49e755b8649c596627a79f3c0bbc1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f2494aa644eb4f72af51412cfbd75eac, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e6d24f68c40e47f088fa57f7558d6437, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/8ca0540e4f574add8ccc8d2f21533cda, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f875551ffc804b2f9b722669b621f1bf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e8b90b470c804268b7dc3b9cbfcba2e6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/7dc7688e9db94b99b1b89a5b4a5e6d85, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f791d6f905fb4ab7a882652aaa287f84, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/9124051fc45945ffad8d8a18040d42dd, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/0683aaa02b1a45a2a7648972c49d7ee5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/7e44b087d2484131a75c06f5aa898248, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/856ddb7953d6458083b54a27965419d4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/59e2ef1aa0524aa2ab1f05e617b5c77f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/c9da0de3084d4b3cbe4056c8c6b15f2e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/802224bfbeed4bb2a6a0181b6ca3d3e0, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e976cf271e6145c582e17df17530e991, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/5a85a230529d4c53818d2cb4e4073b48, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/8caff7c0080e4b70b7c7bf0fea62c4e6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/36c51f6809064c75804ee1cfa6b38ae5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/8f0001e2ba184c7eab75f3922f50fa1c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/2e7d195fc5b14d6694af6b836c31d0a8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/0563f99434334ed78b5430ec0e13bf58, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/650bbf9d1ccb4143aa9fbab65b786401, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f0e08f0a18ba4cbd929a0f6aa5c4fc56, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/cbecd4d09e55411583942de50d039d84, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/59e97c7f71dd40db93f5bfd3e28684fa] to archive 2024-12-02T05:19:16,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:19:16,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/04a49e755b8649c596627a79f3c0bbc1 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/04a49e755b8649c596627a79f3c0bbc1 2024-12-02T05:19:16,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f2494aa644eb4f72af51412cfbd75eac to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f2494aa644eb4f72af51412cfbd75eac 2024-12-02T05:19:16,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e6d24f68c40e47f088fa57f7558d6437 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e6d24f68c40e47f088fa57f7558d6437 2024-12-02T05:19:16,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/8ca0540e4f574add8ccc8d2f21533cda to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/8ca0540e4f574add8ccc8d2f21533cda 2024-12-02T05:19:16,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f875551ffc804b2f9b722669b621f1bf to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f875551ffc804b2f9b722669b621f1bf 2024-12-02T05:19:16,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e8b90b470c804268b7dc3b9cbfcba2e6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e8b90b470c804268b7dc3b9cbfcba2e6 2024-12-02T05:19:16,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/7dc7688e9db94b99b1b89a5b4a5e6d85 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/7dc7688e9db94b99b1b89a5b4a5e6d85 2024-12-02T05:19:16,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f791d6f905fb4ab7a882652aaa287f84 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f791d6f905fb4ab7a882652aaa287f84 2024-12-02T05:19:16,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/9124051fc45945ffad8d8a18040d42dd to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/9124051fc45945ffad8d8a18040d42dd 2024-12-02T05:19:16,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/0683aaa02b1a45a2a7648972c49d7ee5 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/0683aaa02b1a45a2a7648972c49d7ee5 2024-12-02T05:19:16,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/7e44b087d2484131a75c06f5aa898248 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/7e44b087d2484131a75c06f5aa898248 2024-12-02T05:19:16,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/856ddb7953d6458083b54a27965419d4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/856ddb7953d6458083b54a27965419d4 2024-12-02T05:19:16,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/59e2ef1aa0524aa2ab1f05e617b5c77f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/59e2ef1aa0524aa2ab1f05e617b5c77f 2024-12-02T05:19:16,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/c9da0de3084d4b3cbe4056c8c6b15f2e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/c9da0de3084d4b3cbe4056c8c6b15f2e 2024-12-02T05:19:16,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/802224bfbeed4bb2a6a0181b6ca3d3e0 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/802224bfbeed4bb2a6a0181b6ca3d3e0 2024-12-02T05:19:16,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e976cf271e6145c582e17df17530e991 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/e976cf271e6145c582e17df17530e991 2024-12-02T05:19:16,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/5a85a230529d4c53818d2cb4e4073b48 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/5a85a230529d4c53818d2cb4e4073b48 2024-12-02T05:19:16,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/8caff7c0080e4b70b7c7bf0fea62c4e6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/8caff7c0080e4b70b7c7bf0fea62c4e6 2024-12-02T05:19:16,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/36c51f6809064c75804ee1cfa6b38ae5 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/36c51f6809064c75804ee1cfa6b38ae5 2024-12-02T05:19:16,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/8f0001e2ba184c7eab75f3922f50fa1c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/8f0001e2ba184c7eab75f3922f50fa1c 2024-12-02T05:19:16,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/2e7d195fc5b14d6694af6b836c31d0a8 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/2e7d195fc5b14d6694af6b836c31d0a8 2024-12-02T05:19:16,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/0563f99434334ed78b5430ec0e13bf58 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/0563f99434334ed78b5430ec0e13bf58 2024-12-02T05:19:16,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/650bbf9d1ccb4143aa9fbab65b786401 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/650bbf9d1ccb4143aa9fbab65b786401 2024-12-02T05:19:16,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f0e08f0a18ba4cbd929a0f6aa5c4fc56 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/f0e08f0a18ba4cbd929a0f6aa5c4fc56 2024-12-02T05:19:16,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/cbecd4d09e55411583942de50d039d84 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/cbecd4d09e55411583942de50d039d84 2024-12-02T05:19:16,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/59e97c7f71dd40db93f5bfd3e28684fa to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/59e97c7f71dd40db93f5bfd3e28684fa 2024-12-02T05:19:16,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/3f65037b655a4b81abc608212dcbb9a4, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d7ff732822844ba4bc3ea87720f644a1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/e6dfdcc09adb48c1b98873e3c89098b2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/a956f54c7b8e4095b81bc84a6062ef4f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/9373723c51c54d6a8d3d69a695a90c4a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/5103e3d41bd84fd4a7ad5562a88a8622, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/9cce3c1b62e44b3f85393670868af6be, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/38795e78f5a642479923181781c2ff48, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/278c53564cfe43b9bfd6c1742178d85c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/b5edb82db0ad470480afc8bf9b6a6d9e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d217495f484e4931b6b8e2dfcf50cd66, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/ece0be0e0bf444d78fa74c36f3d31f59, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/f11e101a6f8d4ec1b40d19890b4c4e26, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/34473a2f0544404ab0dd397d50b48ddf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/0e83f7d42e23484390acf4f6c834052a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d6d93500081b4b85ad6834ecadf32c27, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/6ac46af8f4fb4228b374bde3c124d461, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/c3e9ee4894774033aefeaf276839ddbe, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/1d4641d0aa4841e4859700fbc6e4f06b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/c2b185149aa34f5e9df90b8d798021ae, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/94a6eb03e19d41d99b695d86570b3760, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/4f3b4d2a8b574774880a0c03876b1461, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/89f5cc23f08b4223b9a3109de2afac21, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/4147fb0c339742418617f1b2407abf99, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/3e0708d4ae9248bd8ac2d2dd940ad24f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/fbd970fdee7d48a3a34bc1daec1fef1e] to archive 2024-12-02T05:19:16,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:19:16,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/3f65037b655a4b81abc608212dcbb9a4 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/3f65037b655a4b81abc608212dcbb9a4 2024-12-02T05:19:16,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d7ff732822844ba4bc3ea87720f644a1 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d7ff732822844ba4bc3ea87720f644a1 2024-12-02T05:19:16,942 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/e6dfdcc09adb48c1b98873e3c89098b2 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/e6dfdcc09adb48c1b98873e3c89098b2 2024-12-02T05:19:16,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/a956f54c7b8e4095b81bc84a6062ef4f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/a956f54c7b8e4095b81bc84a6062ef4f 2024-12-02T05:19:16,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/9373723c51c54d6a8d3d69a695a90c4a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/9373723c51c54d6a8d3d69a695a90c4a 2024-12-02T05:19:16,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/5103e3d41bd84fd4a7ad5562a88a8622 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/5103e3d41bd84fd4a7ad5562a88a8622 2024-12-02T05:19:16,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/9cce3c1b62e44b3f85393670868af6be to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/9cce3c1b62e44b3f85393670868af6be 2024-12-02T05:19:16,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/38795e78f5a642479923181781c2ff48 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/38795e78f5a642479923181781c2ff48 2024-12-02T05:19:16,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/278c53564cfe43b9bfd6c1742178d85c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/278c53564cfe43b9bfd6c1742178d85c 2024-12-02T05:19:16,951 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/b5edb82db0ad470480afc8bf9b6a6d9e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/b5edb82db0ad470480afc8bf9b6a6d9e 2024-12-02T05:19:16,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d217495f484e4931b6b8e2dfcf50cd66 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d217495f484e4931b6b8e2dfcf50cd66 2024-12-02T05:19:16,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/ece0be0e0bf444d78fa74c36f3d31f59 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/ece0be0e0bf444d78fa74c36f3d31f59 2024-12-02T05:19:16,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/f11e101a6f8d4ec1b40d19890b4c4e26 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/f11e101a6f8d4ec1b40d19890b4c4e26 2024-12-02T05:19:16,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/34473a2f0544404ab0dd397d50b48ddf to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/34473a2f0544404ab0dd397d50b48ddf 2024-12-02T05:19:16,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/0e83f7d42e23484390acf4f6c834052a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/0e83f7d42e23484390acf4f6c834052a 2024-12-02T05:19:16,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d6d93500081b4b85ad6834ecadf32c27 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/d6d93500081b4b85ad6834ecadf32c27 2024-12-02T05:19:16,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/6ac46af8f4fb4228b374bde3c124d461 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/6ac46af8f4fb4228b374bde3c124d461 2024-12-02T05:19:16,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/c3e9ee4894774033aefeaf276839ddbe to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/c3e9ee4894774033aefeaf276839ddbe 2024-12-02T05:19:16,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/1d4641d0aa4841e4859700fbc6e4f06b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/1d4641d0aa4841e4859700fbc6e4f06b 2024-12-02T05:19:16,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/c2b185149aa34f5e9df90b8d798021ae to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/c2b185149aa34f5e9df90b8d798021ae 2024-12-02T05:19:16,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/94a6eb03e19d41d99b695d86570b3760 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/94a6eb03e19d41d99b695d86570b3760 2024-12-02T05:19:16,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/4f3b4d2a8b574774880a0c03876b1461 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/4f3b4d2a8b574774880a0c03876b1461 2024-12-02T05:19:16,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/89f5cc23f08b4223b9a3109de2afac21 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/89f5cc23f08b4223b9a3109de2afac21 2024-12-02T05:19:16,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/4147fb0c339742418617f1b2407abf99 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/4147fb0c339742418617f1b2407abf99 2024-12-02T05:19:16,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/3e0708d4ae9248bd8ac2d2dd940ad24f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/3e0708d4ae9248bd8ac2d2dd940ad24f 2024-12-02T05:19:16,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/fbd970fdee7d48a3a34bc1daec1fef1e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/fbd970fdee7d48a3a34bc1daec1fef1e 2024-12-02T05:19:16,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/cac2948fc2ff4ceab51ceb572661c892, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/a84a2ab733dc46a895601cc44c41e4a2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/d3c71346056b4163ae6be53c07792dd6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f425753dcdd64c20871421a4cfbccecf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e7b3c05a92ca4b21991b098a2536fab1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/a9821ff770df4855bb394359654ced06, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f8956066bb804c52b3b367fb018c2473, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e586ddd19d474d4a9455252b6f5493aa, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/3c9793186d6f4fedac6eed24127ce431, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/5fdea0cb02fa41e48b95fca17e312a0c, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/ad676726d69f4467a6d78488df6a85ae, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/888556972c4646d0bab20c421608c5a5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/8a145ea10b06437bb0e6891073e8e477, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/bedbc4b0784f47769959a50a5cb2ca97, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/5199b7718a5e4c989a376174c616ef27, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/7bfc241b07f14738b790141737f36e38, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/259f84aa106f4be1a1212246f25fc05e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2b5f975f672c48f3a557eeb482fd2835, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2f4a78747464453bae7d9c434fe99a32, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/4c7ebf759df9478e9e94667b3b55aff1, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f621d1c9c7bf4a2dabf7e70c102d1d64, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e0216515387a4f3dbc7921a91a7957d8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2cf20242589941319a22b408fc1132b0, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/0c39a9d7488d427e84ba877a0a84de65, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/737966cf61994682b3d3107584c60df7] to archive 2024-12-02T05:19:16,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:19:16,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/cac2948fc2ff4ceab51ceb572661c892 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/cac2948fc2ff4ceab51ceb572661c892 2024-12-02T05:19:16,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/a84a2ab733dc46a895601cc44c41e4a2 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/a84a2ab733dc46a895601cc44c41e4a2 2024-12-02T05:19:16,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/d3c71346056b4163ae6be53c07792dd6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/d3c71346056b4163ae6be53c07792dd6 2024-12-02T05:19:16,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f425753dcdd64c20871421a4cfbccecf to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f425753dcdd64c20871421a4cfbccecf 2024-12-02T05:19:16,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e7b3c05a92ca4b21991b098a2536fab1 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e7b3c05a92ca4b21991b098a2536fab1 2024-12-02T05:19:16,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/a9821ff770df4855bb394359654ced06 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/a9821ff770df4855bb394359654ced06 2024-12-02T05:19:16,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f8956066bb804c52b3b367fb018c2473 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f8956066bb804c52b3b367fb018c2473 2024-12-02T05:19:16,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e586ddd19d474d4a9455252b6f5493aa to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e586ddd19d474d4a9455252b6f5493aa 2024-12-02T05:19:16,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/3c9793186d6f4fedac6eed24127ce431 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/3c9793186d6f4fedac6eed24127ce431 2024-12-02T05:19:16,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/5fdea0cb02fa41e48b95fca17e312a0c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/5fdea0cb02fa41e48b95fca17e312a0c 2024-12-02T05:19:16,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/ad676726d69f4467a6d78488df6a85ae to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/ad676726d69f4467a6d78488df6a85ae 2024-12-02T05:19:16,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/888556972c4646d0bab20c421608c5a5 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/888556972c4646d0bab20c421608c5a5 2024-12-02T05:19:16,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/8a145ea10b06437bb0e6891073e8e477 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/8a145ea10b06437bb0e6891073e8e477 2024-12-02T05:19:16,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/bedbc4b0784f47769959a50a5cb2ca97 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/bedbc4b0784f47769959a50a5cb2ca97 2024-12-02T05:19:16,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/5199b7718a5e4c989a376174c616ef27 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/5199b7718a5e4c989a376174c616ef27 2024-12-02T05:19:16,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/7bfc241b07f14738b790141737f36e38 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/7bfc241b07f14738b790141737f36e38 2024-12-02T05:19:16,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/259f84aa106f4be1a1212246f25fc05e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/259f84aa106f4be1a1212246f25fc05e 2024-12-02T05:19:16,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2b5f975f672c48f3a557eeb482fd2835 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2b5f975f672c48f3a557eeb482fd2835 2024-12-02T05:19:16,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2f4a78747464453bae7d9c434fe99a32 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2f4a78747464453bae7d9c434fe99a32 2024-12-02T05:19:16,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/4c7ebf759df9478e9e94667b3b55aff1 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/4c7ebf759df9478e9e94667b3b55aff1 2024-12-02T05:19:16,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f621d1c9c7bf4a2dabf7e70c102d1d64 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/f621d1c9c7bf4a2dabf7e70c102d1d64 2024-12-02T05:19:16,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e0216515387a4f3dbc7921a91a7957d8 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/e0216515387a4f3dbc7921a91a7957d8 2024-12-02T05:19:16,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2cf20242589941319a22b408fc1132b0 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/2cf20242589941319a22b408fc1132b0 2024-12-02T05:19:16,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/0c39a9d7488d427e84ba877a0a84de65 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/0c39a9d7488d427e84ba877a0a84de65 2024-12-02T05:19:16,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/737966cf61994682b3d3107584c60df7 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/737966cf61994682b3d3107584c60df7 2024-12-02T05:19:16,995 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/recovered.edits/513.seqid, newMaxSeqId=513, maxSeqId=1 2024-12-02T05:19:16,996 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9. 2024-12-02T05:19:16,996 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for be075553e9fdd4e7a03804f307b7c3f9: Waiting for close lock at 1733116755807Running coprocessor pre-close hooks at 1733116755807Disabling compacts and flushes for region at 1733116755807Disabling writes for close at 1733116756903 (+1096 ms)Writing region close event to WAL at 1733116756993 (+90 ms)Running coprocessor post-close hooks at 1733116756996 (+3 ms)Closed at 1733116756996 2024-12-02T05:19:16,997 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:16,998 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=be075553e9fdd4e7a03804f307b7c3f9, regionState=CLOSED 2024-12-02T05:19:16,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:19:17,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-02T05:19:17,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure be075553e9fdd4e7a03804f307b7c3f9, server=f49461030c22,46495,1733116594459 in 1.3440 sec 2024-12-02T05:19:17,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-02T05:19:17,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be075553e9fdd4e7a03804f307b7c3f9, UNASSIGN in 1.3480 sec 2024-12-02T05:19:17,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-02T05:19:17,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3520 sec 2024-12-02T05:19:17,005 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116757004"}]},"ts":"1733116757004"} 2024-12-02T05:19:17,006 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T05:19:17,006 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T05:19:17,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3640 sec 2024-12-02T05:19:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T05:19:17,799 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:17,799 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T05:19:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:17,803 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T05:19:17,804 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:17,808 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:17,811 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/recovered.edits] 2024-12-02T05:19:17,814 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/7aac5eb51fed4be29513834d60dcedc8 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/7aac5eb51fed4be29513834d60dcedc8 2024-12-02T05:19:17,816 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/bcd9e9d1f8eb47ce90d3ef4ab60b9bac to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/bcd9e9d1f8eb47ce90d3ef4ab60b9bac 2024-12-02T05:19:17,817 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/d17df7f68e3848feaab14d0a3c3d311b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/A/d17df7f68e3848feaab14d0a3c3d311b 2024-12-02T05:19:17,818 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/07acc76a8eb74cdbbcb48a81bca86566 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/07acc76a8eb74cdbbcb48a81bca86566 2024-12-02T05:19:17,819 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/338c47471d2c43cd9feac9753b4618a1 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/338c47471d2c43cd9feac9753b4618a1 2024-12-02T05:19:17,820 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/3616ef64f5084326a2dce64015eb90cc to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/B/3616ef64f5084326a2dce64015eb90cc 2024-12-02T05:19:17,821 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/5a36f72a46734bbd9abc87207dbf383f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/5a36f72a46734bbd9abc87207dbf383f 2024-12-02T05:19:17,822 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/ad2a290371a04d71abffd0ffb01d113b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/ad2a290371a04d71abffd0ffb01d113b 2024-12-02T05:19:17,823 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/ad405d37e1da4baf9bdd2edd36362bc7 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/C/ad405d37e1da4baf9bdd2edd36362bc7 2024-12-02T05:19:17,825 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/recovered.edits/513.seqid to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9/recovered.edits/513.seqid 2024-12-02T05:19:17,825 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/be075553e9fdd4e7a03804f307b7c3f9 2024-12-02T05:19:17,825 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T05:19:17,827 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:17,829 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T05:19:17,830 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T05:19:17,831 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:17,831 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T05:19:17,831 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733116757831"}]},"ts":"9223372036854775807"} 2024-12-02T05:19:17,833 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T05:19:17,833 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => be075553e9fdd4e7a03804f307b7c3f9, NAME => 'TestAcidGuarantees,,1733116729112.be075553e9fdd4e7a03804f307b7c3f9.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T05:19:17,833 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T05:19:17,833 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733116757833"}]},"ts":"9223372036854775807"} 2024-12-02T05:19:17,834 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T05:19:17,835 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:17,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 35 msec 2024-12-02T05:19:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T05:19:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T05:19:17,911 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:17,922 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 248) - Thread LEAK? -, OpenFileDescriptor=445 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=582 (was 669), ProcessCount=11 (was 11), AvailableMemoryMB=5814 (was 5821) 2024-12-02T05:19:17,932 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=582, ProcessCount=11, AvailableMemoryMB=5814 2024-12-02T05:19:17,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T05:19:17,934 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T05:19:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:17,936 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T05:19:17,936 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:17,936 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-12-02T05:19:17,937 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T05:19:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-02T05:19:17,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742330_1506 (size=1061) 2024-12-02T05:19:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-02T05:19:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-02T05:19:18,345 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c 2024-12-02T05:19:18,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742331_1507 (size=53) 2024-12-02T05:19:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-02T05:19:18,754 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:19:18,754 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1160304dbb0b6d1b48d2c66c029eb555, disabling compactions & flushes 2024-12-02T05:19:18,754 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:18,754 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:18,754 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. after waiting 0 ms 2024-12-02T05:19:18,754 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:18,754 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:18,754 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1160304dbb0b6d1b48d2c66c029eb555: Waiting for close lock at 1733116758754Disabling compacts and flushes for region at 1733116758754Disabling writes for close at 1733116758754Writing region close event to WAL at 1733116758754Closed at 1733116758754 2024-12-02T05:19:18,756 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T05:19:18,756 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733116758756"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733116758756"}]},"ts":"1733116758756"} 2024-12-02T05:19:18,759 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T05:19:18,760 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T05:19:18,760 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116758760"}]},"ts":"1733116758760"} 2024-12-02T05:19:18,763 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T05:19:18,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1160304dbb0b6d1b48d2c66c029eb555, ASSIGN}] 2024-12-02T05:19:18,765 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1160304dbb0b6d1b48d2c66c029eb555, ASSIGN 2024-12-02T05:19:18,766 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1160304dbb0b6d1b48d2c66c029eb555, ASSIGN; state=OFFLINE, location=f49461030c22,46495,1733116594459; forceNewPlan=false, retain=false 2024-12-02T05:19:18,917 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1160304dbb0b6d1b48d2c66c029eb555, regionState=OPENING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:19:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-02T05:19:19,112 INFO [AsyncFSWAL-0-hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c-prefix:f49461030c22,46495,1733116594459.meta {}] wal.AbstractFSWAL(1368): Slow sync cost: 193 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34519,DS-5a5c0e22-a56e-4f32-ae1e-a90082acf4bf,DISK]] 2024-12-02T05:19:19,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1160304dbb0b6d1b48d2c66c029eb555, ASSIGN because future has completed 2024-12-02T05:19:19,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:19:19,270 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:19,270 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} 2024-12-02T05:19:19,271 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:19,271 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:19:19,271 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:19,271 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:19,272 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:19,274 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:19:19,274 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1160304dbb0b6d1b48d2c66c029eb555 columnFamilyName A 2024-12-02T05:19:19,274 DEBUG [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:19,275 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.HStore(327): Store=1160304dbb0b6d1b48d2c66c029eb555/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:19:19,275 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:19,277 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:19:19,277 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1160304dbb0b6d1b48d2c66c029eb555 columnFamilyName B 2024-12-02T05:19:19,277 DEBUG [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:19,278 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.HStore(327): Store=1160304dbb0b6d1b48d2c66c029eb555/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:19:19,278 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:19,279 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:19:19,279 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1160304dbb0b6d1b48d2c66c029eb555 columnFamilyName C 2024-12-02T05:19:19,280 DEBUG [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:19,280 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.HStore(327): Store=1160304dbb0b6d1b48d2c66c029eb555/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:19:19,280 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:19,281 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:19,281 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:19,282 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:19,283 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:19,283 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:19,284 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T05:19:19,285 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:19,287 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T05:19:19,288 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened 1160304dbb0b6d1b48d2c66c029eb555; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73965459, jitterRate=0.10217122733592987}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T05:19:19,288 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:19,289 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for 1160304dbb0b6d1b48d2c66c029eb555: Running coprocessor pre-open hook at 1733116759271Writing region info on filesystem at 1733116759271Initializing all the Stores at 1733116759272 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116759272Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116759272Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116759272Cleaning up temporary data from old regions at 1733116759283 (+11 ms)Running coprocessor post-open hooks at 1733116759288 (+5 ms)Region opened successfully at 1733116759288 2024-12-02T05:19:19,289 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., pid=144, masterSystemTime=1733116759267 2024-12-02T05:19:19,291 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:19,291 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:19,292 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1160304dbb0b6d1b48d2c66c029eb555, regionState=OPEN, openSeqNum=2, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:19:19,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:19:19,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-02T05:19:19,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 in 181 msec 2024-12-02T05:19:19,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-02T05:19:19,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1160304dbb0b6d1b48d2c66c029eb555, ASSIGN in 532 msec 2024-12-02T05:19:19,298 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T05:19:19,298 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116759298"}]},"ts":"1733116759298"} 2024-12-02T05:19:19,299 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T05:19:19,299 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T05:19:19,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3650 sec 2024-12-02T05:19:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-02T05:19:20,088 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:20,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348042de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:20,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:19:20,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:19:20,092 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:19:20,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:19:20,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:19:20,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e3ad985, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:20,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:19:20,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:19:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:20,095 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:19:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c9dd8f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:20,097 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:19:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:19:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:19:20,099 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:19:20,100 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T05:19:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is f49461030c22,42731,1733116593060 2024-12-02T05:19:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4892481 2024-12-02T05:19:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T05:19:20,102 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T05:19:20,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T05:19:20,103 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T05:19:20,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:20,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742332_1508 (size=1097) 2024-12-02T05:19:20,514 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T05:19:20,514 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T05:19:20,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T05:19:20,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1160304dbb0b6d1b48d2c66c029eb555, REOPEN/MOVE}] 2024-12-02T05:19:20,520 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1160304dbb0b6d1b48d2c66c029eb555, REOPEN/MOVE 2024-12-02T05:19:20,521 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=1160304dbb0b6d1b48d2c66c029eb555, regionState=CLOSING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:19:20,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1160304dbb0b6d1b48d2c66c029eb555, REOPEN/MOVE because future has completed 2024-12-02T05:19:20,522 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T05:19:20,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:19:20,676 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:20,676 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T05:19:20,676 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 1160304dbb0b6d1b48d2c66c029eb555, disabling compactions & flushes 2024-12-02T05:19:20,676 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:20,677 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:20,677 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. after waiting 0 ms 2024-12-02T05:19:20,677 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:20,683 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T05:19:20,684 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:20,684 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 1160304dbb0b6d1b48d2c66c029eb555: Waiting for close lock at 1733116760676Running coprocessor pre-close hooks at 1733116760676Disabling compacts and flushes for region at 1733116760676Disabling writes for close at 1733116760677 (+1 ms)Writing region close event to WAL at 1733116760678 (+1 ms)Running coprocessor post-close hooks at 1733116760684 (+6 ms)Closed at 1733116760684 2024-12-02T05:19:20,684 WARN [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: 1160304dbb0b6d1b48d2c66c029eb555 to self. 2024-12-02T05:19:20,687 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:20,688 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=1160304dbb0b6d1b48d2c66c029eb555, regionState=CLOSED 2024-12-02T05:19:20,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:19:20,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-02T05:19:20,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 in 169 msec 2024-12-02T05:19:20,696 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1160304dbb0b6d1b48d2c66c029eb555, REOPEN/MOVE; state=CLOSED, location=f49461030c22,46495,1733116594459; forceNewPlan=false, retain=true 2024-12-02T05:19:20,846 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=1160304dbb0b6d1b48d2c66c029eb555, regionState=OPENING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:19:20,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1160304dbb0b6d1b48d2c66c029eb555, REOPEN/MOVE because future has completed 2024-12-02T05:19:20,848 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:19:21,002 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:21,002 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} 2024-12-02T05:19:21,002 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,002 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T05:19:21,003 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,003 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,004 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,004 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:19:21,004 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1160304dbb0b6d1b48d2c66c029eb555 columnFamilyName A 2024-12-02T05:19:21,005 DEBUG [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:21,005 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.HStore(327): Store=1160304dbb0b6d1b48d2c66c029eb555/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:19:21,006 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,006 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:19:21,006 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1160304dbb0b6d1b48d2c66c029eb555 columnFamilyName B 2024-12-02T05:19:21,006 DEBUG [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:21,006 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.HStore(327): Store=1160304dbb0b6d1b48d2c66c029eb555/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:19:21,006 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,007 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T05:19:21,007 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1160304dbb0b6d1b48d2c66c029eb555 columnFamilyName C 2024-12-02T05:19:21,007 DEBUG [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:21,007 INFO [StoreOpener-1160304dbb0b6d1b48d2c66c029eb555-1 {}] regionserver.HStore(327): Store=1160304dbb0b6d1b48d2c66c029eb555/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T05:19:21,007 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:21,007 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,008 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,009 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,010 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,010 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,010 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T05:19:21,011 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,012 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened 1160304dbb0b6d1b48d2c66c029eb555; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59229697, jitterRate=-0.11740873754024506}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T05:19:21,012 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,012 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for 1160304dbb0b6d1b48d2c66c029eb555: Running coprocessor pre-open hook at 1733116761003Writing region info on filesystem at 1733116761003Initializing all the Stores at 1733116761003Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116761003Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116761003Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733116761003Cleaning up temporary data from old regions at 1733116761010 (+7 ms)Running coprocessor post-open hooks at 1733116761012 (+2 ms)Region opened successfully at 1733116761012 2024-12-02T05:19:21,013 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., pid=149, masterSystemTime=1733116760999 2024-12-02T05:19:21,015 DEBUG [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:21,015 INFO [RS_OPEN_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:21,015 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=1160304dbb0b6d1b48d2c66c029eb555, regionState=OPEN, openSeqNum=5, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:19:21,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-12-02T05:19:21,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 in 168 msec 2024-12-02T05:19:21,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-02T05:19:21,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1160304dbb0b6d1b48d2c66c029eb555, REOPEN/MOVE in 499 msec 2024-12-02T05:19:21,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-02T05:19:21,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 504 msec 2024-12-02T05:19:21,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 919 msec 2024-12-02T05:19:21,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T05:19:21,024 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:21,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:19:21,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:19:21,025 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:19:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:19:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:19:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ca8b1bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:19:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:19:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:21,026 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:19:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62741fa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:19:21,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:19:21,028 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:19:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:19:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:19:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ba01a8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:19:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:19:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:21,029 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:19:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c477898, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,030 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:19:21,030 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:19:21,030 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:19:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:19:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:19:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6495cfdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:19:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:19:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:21,031 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:19:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43e7648, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:19:21,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:19:21,033 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:19:21,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:19:21,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:19:21,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2127afe8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:19:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:19:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:21,034 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:19:21,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fec02d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,035 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:19:21,035 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:19:21,036 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:19:21,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:19:21,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:19:21,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c1d6e08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:19:21,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:19:21,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:21,037 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:19:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f427fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:19:21,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:19:21,038 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:19:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:19:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:19:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff99f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:19:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:19:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:21,039 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:19:21,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44cdc5e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,040 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:19:21,040 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:19:21,041 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:19:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:19:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:19:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c292fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:19:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:19:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:21,042 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:19:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f5e5657, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:19:21,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:19:21,044 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:19:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:19:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:19:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@abf93e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:19:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:19:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:21,045 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:19:21,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64019c90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,046 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:19:21,046 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:19:21,047 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:19:21,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:19:21,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:19:21,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fafa7de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:19:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:19:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:21,048 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:19:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@659de5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a2803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,050 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f49461030c22,42731,-1 for getting cluster id 2024-12-02T05:19:21,050 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T05:19:21,050 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21' 2024-12-02T05:19:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T05:19:21,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e24d6dfb-61d5-4bfb-b7d6-1aa730fd1d21" 2024-12-02T05:19:21,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@193c0602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f49461030c22,42731,-1] 2024-12-02T05:19:21,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T05:19:21,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:21,052 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T05:19:21,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d1a87ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T05:19:21,053 DEBUG [Thread-2303 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:19:21,053 DEBUG [Thread-2297 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:19:21,053 DEBUG [Thread-2301 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:19:21,053 DEBUG [Thread-2294 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:19:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:19:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:19:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:19:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:19:21,054 DEBUG [Thread-2286 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:19:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:19:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:19:21,054 DEBUG [Thread-2288 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:19:21,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:19:21,055 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:19:21,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:19:21,055 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:21,055 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:19:21,055 DEBUG [Thread-2292 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:19:21,055 DEBUG [Thread-2305 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:19:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:19:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:19:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:19:21,056 DEBUG [Thread-2290 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:19:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:19:21,056 DEBUG [Thread-2299 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T05:19:21,056 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:19:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:19:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:19:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:19:21,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:19:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:19:21,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:19:21,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:19:21,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:19:21,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:19:21,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:19:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-02T05:19:21,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:19:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:19:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:19:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:19:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:19:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T05:19:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:19:21,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:19:21,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:19:21,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:19:21,061 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:21,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:19:21,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f49461030c22,46495,1733116594459, seqNum=-1] 2024-12-02T05:19:21,063 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:21,063 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T05:19:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:19:21,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T05:19:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5] 2024-12-02T05:19:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T05:19:21,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:21,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:21,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116761075, max timestamp=1733116761090Num uniques 859; 2024-12-02T05:19:21,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:19:21,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116761075, max timestamp=1733116761090Num uniques 859; 2024-12-02T05:19:21,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733116761075, max timestamp=1733116761090Num uniques 859; 2024-12-02T05:19:21,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:19:21,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T05:19:21,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020c25218d40b04533a8a30c0041d64d33_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116761066/Put/seqid=0 2024-12-02T05:19:21,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116821105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116821106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116821111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116821111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116821111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742333_1509 (size=12154) 2024-12-02T05:19:21,127 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:21,133 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020c25218d40b04533a8a30c0041d64d33_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020c25218d40b04533a8a30c0041d64d33_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,134 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/5327f319f2944748b41d27303a63e6c2, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:21,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/5327f319f2944748b41d27303a63e6c2 is 175, key is test_row_0/A:col10/1733116761066/Put/seqid=0 2024-12-02T05:19:21,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742334_1510 (size=30955) 2024-12-02T05:19:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T05:19:21,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T05:19:21,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:21,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:21,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:21,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:21,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:21,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116821218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116821218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116821218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116821218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116821218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T05:19:21,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:21,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:21,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:21,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:21,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:21,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T05:19:21,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116821427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116821427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116821427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116821427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116821427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T05:19:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:21,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:21,545 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/5327f319f2944748b41d27303a63e6c2 2024-12-02T05:19:21,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/96c8c1079b6941c697200942cff2098e is 50, key is test_row_0/B:col10/1733116761066/Put/seqid=0 2024-12-02T05:19:21,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742335_1511 (size=12001) 2024-12-02T05:19:21,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/96c8c1079b6941c697200942cff2098e 2024-12-02T05:19:21,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/03033c5059774f4fab5cdeccbfc16e53 is 50, key is test_row_0/C:col10/1733116761066/Put/seqid=0 2024-12-02T05:19:21,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742336_1512 (size=12001) 2024-12-02T05:19:21,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/03033c5059774f4fab5cdeccbfc16e53 2024-12-02T05:19:21,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/5327f319f2944748b41d27303a63e6c2 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/5327f319f2944748b41d27303a63e6c2 2024-12-02T05:19:21,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/5327f319f2944748b41d27303a63e6c2, entries=150, sequenceid=15, filesize=30.2 K 2024-12-02T05:19:21,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/96c8c1079b6941c697200942cff2098e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/96c8c1079b6941c697200942cff2098e 2024-12-02T05:19:21,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/96c8c1079b6941c697200942cff2098e, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T05:19:21,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/03033c5059774f4fab5cdeccbfc16e53 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/03033c5059774f4fab5cdeccbfc16e53 2024-12-02T05:19:21,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/03033c5059774f4fab5cdeccbfc16e53, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T05:19:21,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1160304dbb0b6d1b48d2c66c029eb555 in 546ms, sequenceid=15, compaction requested=false 2024-12-02T05:19:21,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:21,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T05:19:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:21,675 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-02T05:19:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:21,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120247bb747234db4044b1d8a685a372e217_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116761108/Put/seqid=0 2024-12-02T05:19:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T05:19:21,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742337_1513 (size=12154) 2024-12-02T05:19:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:21,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:21,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116821741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116821744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116821745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116821746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116821746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116821858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116821858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116821858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116821858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116821858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116822067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116822068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116822068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116822068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:22,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116822068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:22,099 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120247bb747234db4044b1d8a685a372e217_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120247bb747234db4044b1d8a685a372e217_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:22,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/85441ae48cc34a269a0100cae6e733f8, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:22,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/85441ae48cc34a269a0100cae6e733f8 is 175, key is test_row_0/A:col10/1733116761108/Put/seqid=0 2024-12-02T05:19:22,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742338_1514 (size=30955) 2024-12-02T05:19:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T05:19:22,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116822377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116822378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:22,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116822378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116822378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116822378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,504 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/85441ae48cc34a269a0100cae6e733f8 2024-12-02T05:19:22,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/381f6649ff2b40e09b82b31132c081fd is 50, key is test_row_0/B:col10/1733116761108/Put/seqid=0 2024-12-02T05:19:22,513 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T05:19:22,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742339_1515 (size=12001) 2024-12-02T05:19:22,540 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/381f6649ff2b40e09b82b31132c081fd 2024-12-02T05:19:22,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/6b2ba22a5aa040658584aa97d0f52095 is 50, key is test_row_0/C:col10/1733116761108/Put/seqid=0 2024-12-02T05:19:22,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742340_1516 (size=12001) 2024-12-02T05:19:22,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116822887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116822887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116822887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:22,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116822888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116822888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:22,957 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/6b2ba22a5aa040658584aa97d0f52095 2024-12-02T05:19:22,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/85441ae48cc34a269a0100cae6e733f8 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/85441ae48cc34a269a0100cae6e733f8 2024-12-02T05:19:22,963 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/85441ae48cc34a269a0100cae6e733f8, entries=150, sequenceid=50, filesize=30.2 K 2024-12-02T05:19:22,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/381f6649ff2b40e09b82b31132c081fd as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/381f6649ff2b40e09b82b31132c081fd 2024-12-02T05:19:22,967 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/381f6649ff2b40e09b82b31132c081fd, entries=150, sequenceid=50, filesize=11.7 K 2024-12-02T05:19:22,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/6b2ba22a5aa040658584aa97d0f52095 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6b2ba22a5aa040658584aa97d0f52095 2024-12-02T05:19:22,971 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6b2ba22a5aa040658584aa97d0f52095, entries=150, sequenceid=50, filesize=11.7 K 2024-12-02T05:19:22,972 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 1160304dbb0b6d1b48d2c66c029eb555 in 1296ms, sequenceid=50, compaction requested=false 2024-12-02T05:19:22,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:22,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:22,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-02T05:19:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-02T05:19:22,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-02T05:19:22,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9100 sec 2024-12-02T05:19:22,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.9200 sec 2024-12-02T05:19:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T05:19:23,208 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:23,208 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-02T05:19:23,210 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T05:19:23,210 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:23,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T05:19:23,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T05:19:23,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:23,363 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T05:19:23,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:23,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:23,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:23,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:23,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:23,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:23,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202593b6bce48b344308bfdcc9b2f5f5ab6_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116761745/Put/seqid=0 2024-12-02T05:19:23,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742341_1517 (size=12154) 2024-12-02T05:19:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T05:19:23,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:23,777 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202593b6bce48b344308bfdcc9b2f5f5ab6_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202593b6bce48b344308bfdcc9b2f5f5ab6_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:23,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/a809c8c8a6514ce3bc9a691965ab72e0, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:23,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/a809c8c8a6514ce3bc9a691965ab72e0 is 175, key is test_row_0/A:col10/1733116761745/Put/seqid=0 2024-12-02T05:19:23,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742342_1518 (size=30955) 2024-12-02T05:19:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T05:19:24,248 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_955742804_22 at /127.0.0.1:49286 [Receiving block BP-16702269-172.17.0.2-1733116583194:blk_1073741833_1009] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 348ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/cluster_01f97ee8-23a4-f5a0-8cda-7a5425401614/data/data1/, blockId=1073741833, seqno=1992 2024-12-02T05:19:24,248 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/a809c8c8a6514ce3bc9a691965ab72e0 2024-12-02T05:19:24,249 INFO [AsyncFSWAL-0-hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c-prefix:f49461030c22,46495,1733116594459 {}] wal.AbstractFSWAL(1368): Slow sync cost: 349 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34519,DS-5a5c0e22-a56e-4f32-ae1e-a90082acf4bf,DISK]] 2024-12-02T05:19:24,249 INFO [AsyncFSWAL-0-hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c-prefix:f49461030c22,46495,1733116594459 {}] wal.AbstractFSWAL(1368): Slow sync cost: 349 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34519,DS-5a5c0e22-a56e-4f32-ae1e-a90082acf4bf,DISK]] 2024-12-02T05:19:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:24,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:24,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/36b14806b9ed47f7a7582d1779d0b99b is 50, key is test_row_0/B:col10/1733116761745/Put/seqid=0 2024-12-02T05:19:24,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116824269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116824269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116824271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,276 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,276 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,276 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742343_1519 (size=12001) 2024-12-02T05:19:24,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116824273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116824274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,280 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/36b14806b9ed47f7a7582d1779d0b99b 2024-12-02T05:19:24,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/b7d33be043f547ce8b70719e2d523a2f is 50, key is test_row_0/C:col10/1733116761745/Put/seqid=0 2024-12-02T05:19:24,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742344_1520 (size=12001) 2024-12-02T05:19:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T05:19:24,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116824378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116824378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116824387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116824387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116824387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116824587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116824587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116824597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116824598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116824598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,698 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/b7d33be043f547ce8b70719e2d523a2f 2024-12-02T05:19:24,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/a809c8c8a6514ce3bc9a691965ab72e0 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/a809c8c8a6514ce3bc9a691965ab72e0 2024-12-02T05:19:24,705 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/a809c8c8a6514ce3bc9a691965ab72e0, entries=150, sequenceid=62, filesize=30.2 K 2024-12-02T05:19:24,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/36b14806b9ed47f7a7582d1779d0b99b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/36b14806b9ed47f7a7582d1779d0b99b 2024-12-02T05:19:24,709 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/36b14806b9ed47f7a7582d1779d0b99b, entries=150, sequenceid=62, filesize=11.7 K 2024-12-02T05:19:24,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/b7d33be043f547ce8b70719e2d523a2f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/b7d33be043f547ce8b70719e2d523a2f 2024-12-02T05:19:24,714 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/b7d33be043f547ce8b70719e2d523a2f, entries=150, sequenceid=62, filesize=11.7 K 2024-12-02T05:19:24,715 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1160304dbb0b6d1b48d2c66c029eb555 in 1353ms, sequenceid=62, compaction requested=true 2024-12-02T05:19:24,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:24,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:24,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-02T05:19:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-02T05:19:24,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-02T05:19:24,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5070 sec 2024-12-02T05:19:24,720 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.5110 sec 2024-12-02T05:19:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:24,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T05:19:24,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:24,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:24,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:24,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:24,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:24,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:24,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202558341c09fd84582925ac73de2fe8946_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116764272/Put/seqid=0 2024-12-02T05:19:24,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116824911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742345_1521 (size=17034) 2024-12-02T05:19:24,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116824913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116824914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116824914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116824915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116825017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116825027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:25,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116825028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116825028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:25,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116825028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:25,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116825228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116825237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:25,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116825237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116825237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116825237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,318 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:25,321 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202558341c09fd84582925ac73de2fe8946_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202558341c09fd84582925ac73de2fe8946_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:25,322 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/640fbae191dd449da5b6066786812895, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:25,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/640fbae191dd449da5b6066786812895 is 175, key is test_row_0/A:col10/1733116764272/Put/seqid=0 2024-12-02T05:19:25,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742346_1522 (size=48139) 2024-12-02T05:19:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T05:19:25,357 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:25,358 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-02T05:19:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T05:19:25,359 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:25,360 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:25,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T05:19:25,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-02T05:19:25,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:25,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:25,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:25,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:25,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:25,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116825538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116825547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116825547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116825547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:25,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116825547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:25,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-02T05:19:25,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:25,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:25,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:25,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:25,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:25,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T05:19:25,730 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/640fbae191dd449da5b6066786812895 2024-12-02T05:19:25,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/e63354599f3e4dc28a50bcdc8b9f4edf is 50, key is test_row_0/B:col10/1733116764272/Put/seqid=0 2024-12-02T05:19:25,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742347_1523 (size=12001) 2024-12-02T05:19:25,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/e63354599f3e4dc28a50bcdc8b9f4edf 2024-12-02T05:19:25,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/97a032671ade4dfc9801515ff17e8e31 is 50, key is test_row_0/C:col10/1733116764272/Put/seqid=0 2024-12-02T05:19:25,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742348_1524 (size=12001) 2024-12-02T05:19:25,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-02T05:19:25,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:25,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:25,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:25,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:25,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:25,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-02T05:19:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:25,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T05:19:26,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116826047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:26,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116826058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:26,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116826058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:26,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116826058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:26,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:26,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:26,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:26,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116826067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:26,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-02T05:19:26,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:26,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:26,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:26,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:26,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/97a032671ade4dfc9801515ff17e8e31 2024-12-02T05:19:26,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/640fbae191dd449da5b6066786812895 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/640fbae191dd449da5b6066786812895 2024-12-02T05:19:26,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/640fbae191dd449da5b6066786812895, entries=250, sequenceid=84, filesize=47.0 K 2024-12-02T05:19:26,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/e63354599f3e4dc28a50bcdc8b9f4edf as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/e63354599f3e4dc28a50bcdc8b9f4edf 2024-12-02T05:19:26,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/e63354599f3e4dc28a50bcdc8b9f4edf, entries=150, sequenceid=84, filesize=11.7 K 2024-12-02T05:19:26,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/97a032671ade4dfc9801515ff17e8e31 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/97a032671ade4dfc9801515ff17e8e31 2024-12-02T05:19:26,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/97a032671ade4dfc9801515ff17e8e31, entries=150, sequenceid=84, filesize=11.7 K 2024-12-02T05:19:26,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for 1160304dbb0b6d1b48d2c66c029eb555 in 1267ms, sequenceid=84, compaction requested=true 2024-12-02T05:19:26,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:26,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:26,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:26,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:19:26,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:26,167 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:19:26,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:19:26,167 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:19:26,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:26,168 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:19:26,168 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:19:26,168 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/B is initiating minor compaction (all files) 2024-12-02T05:19:26,168 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/A is initiating minor compaction (all files) 2024-12-02T05:19:26,168 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/A in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:26,168 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/B in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:26,168 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/5327f319f2944748b41d27303a63e6c2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/85441ae48cc34a269a0100cae6e733f8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/a809c8c8a6514ce3bc9a691965ab72e0, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/640fbae191dd449da5b6066786812895] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=137.7 K 2024-12-02T05:19:26,168 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/96c8c1079b6941c697200942cff2098e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/381f6649ff2b40e09b82b31132c081fd, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/36b14806b9ed47f7a7582d1779d0b99b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/e63354599f3e4dc28a50bcdc8b9f4edf] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=46.9 K 2024-12-02T05:19:26,168 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:26,168 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/5327f319f2944748b41d27303a63e6c2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/85441ae48cc34a269a0100cae6e733f8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/a809c8c8a6514ce3bc9a691965ab72e0, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/640fbae191dd449da5b6066786812895] 2024-12-02T05:19:26,169 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5327f319f2944748b41d27303a63e6c2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733116761065 2024-12-02T05:19:26,169 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 96c8c1079b6941c697200942cff2098e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733116761065 2024-12-02T05:19:26,170 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 381f6649ff2b40e09b82b31132c081fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733116761104 2024-12-02T05:19:26,170 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85441ae48cc34a269a0100cae6e733f8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733116761104 2024-12-02T05:19:26,170 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting a809c8c8a6514ce3bc9a691965ab72e0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733116761744 2024-12-02T05:19:26,170 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b14806b9ed47f7a7582d1779d0b99b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733116761744 2024-12-02T05:19:26,170 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 640fbae191dd449da5b6066786812895, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733116764265 2024-12-02T05:19:26,170 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting e63354599f3e4dc28a50bcdc8b9f4edf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733116764265 2024-12-02T05:19:26,175 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:26,177 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#B#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:26,177 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/7c9c9b27866243d388533733dce121ce is 50, key is test_row_0/B:col10/1733116764272/Put/seqid=0 2024-12-02T05:19:26,179 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120290948e0bdc644a8f852dd944020028ad_1160304dbb0b6d1b48d2c66c029eb555 store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:26,180 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120290948e0bdc644a8f852dd944020028ad_1160304dbb0b6d1b48d2c66c029eb555, store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:26,180 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120290948e0bdc644a8f852dd944020028ad_1160304dbb0b6d1b48d2c66c029eb555 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:26,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742349_1525 (size=12139) 2024-12-02T05:19:26,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742350_1526 (size=4469) 2024-12-02T05:19:26,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-02T05:19:26,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:26,277 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T05:19:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:26,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f5b4a819ea464609ac8ea682b5620c58_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116764913/Put/seqid=0 2024-12-02T05:19:26,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742351_1527 (size=12154) 2024-12-02T05:19:26,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,301 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f5b4a819ea464609ac8ea682b5620c58_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f5b4a819ea464609ac8ea682b5620c58_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:26,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/934cc065d8464b66a36006dbd09c789d, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:26,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/934cc065d8464b66a36006dbd09c789d is 175, key is test_row_0/A:col10/1733116764913/Put/seqid=0 2024-12-02T05:19:26,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742352_1528 (size=30955) 2024-12-02T05:19:26,307 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/934cc065d8464b66a36006dbd09c789d 2024-12-02T05:19:26,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/08795028871847df9741d443e4a0fbd8 is 50, key is test_row_0/B:col10/1733116764913/Put/seqid=0 2024-12-02T05:19:26,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742353_1529 (size=12001) 2024-12-02T05:19:26,317 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/08795028871847df9741d443e4a0fbd8 2024-12-02T05:19:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/114fe259a63a4bf9b4e18dda95a67a82 is 50, key is test_row_0/C:col10/1733116764913/Put/seqid=0 2024-12-02T05:19:26,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742354_1530 (size=12001) 2024-12-02T05:19:26,328 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/114fe259a63a4bf9b4e18dda95a67a82 2024-12-02T05:19:26,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/934cc065d8464b66a36006dbd09c789d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/934cc065d8464b66a36006dbd09c789d 2024-12-02T05:19:26,336 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/934cc065d8464b66a36006dbd09c789d, entries=150, sequenceid=95, filesize=30.2 K 2024-12-02T05:19:26,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/08795028871847df9741d443e4a0fbd8 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/08795028871847df9741d443e4a0fbd8 2024-12-02T05:19:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,339 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/08795028871847df9741d443e4a0fbd8, entries=150, sequenceid=95, filesize=11.7 K 2024-12-02T05:19:26,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/114fe259a63a4bf9b4e18dda95a67a82 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/114fe259a63a4bf9b4e18dda95a67a82 2024-12-02T05:19:26,343 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/114fe259a63a4bf9b4e18dda95a67a82, entries=150, sequenceid=95, filesize=11.7 K 2024-12-02T05:19:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,344 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 1160304dbb0b6d1b48d2c66c029eb555 in 68ms, sequenceid=95, compaction requested=true 2024-12-02T05:19:26,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:26,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:26,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-02T05:19:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-02T05:19:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-02T05:19:26,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 985 msec 2024-12-02T05:19:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 989 msec 2024-12-02T05:19:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T05:19:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,497 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,498 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-02T05:19:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T05:19:26,500 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,500 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:26,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,589 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/7c9c9b27866243d388533733dce121ce as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7c9c9b27866243d388533733dce121ce 2024-12-02T05:19:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,590 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#A#compaction#448 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,591 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/935c8a36d37048d2b1eace19ff640dae is 175, key is test_row_0/A:col10/1733116764272/Put/seqid=0 2024-12-02T05:19:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,595 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/B of 1160304dbb0b6d1b48d2c66c029eb555 into 7c9c9b27866243d388533733dce121ce(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:26,595 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,596 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/B, priority=12, startTime=1733116766167; duration=0sec 2024-12-02T05:19:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,596 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:26,596 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:B 2024-12-02T05:19:26,596 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T05:19:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,597 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T05:19:26,597 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/C is initiating minor compaction (all files) 2024-12-02T05:19:26,597 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/C in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:26,597 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/03033c5059774f4fab5cdeccbfc16e53, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6b2ba22a5aa040658584aa97d0f52095, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/b7d33be043f547ce8b70719e2d523a2f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/97a032671ade4dfc9801515ff17e8e31, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/114fe259a63a4bf9b4e18dda95a67a82] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=58.6 K 2024-12-02T05:19:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,598 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 03033c5059774f4fab5cdeccbfc16e53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733116761065 2024-12-02T05:19:26,598 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b2ba22a5aa040658584aa97d0f52095, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733116761104 2024-12-02T05:19:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,598 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting b7d33be043f547ce8b70719e2d523a2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733116761744 2024-12-02T05:19:26,599 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 97a032671ade4dfc9801515ff17e8e31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733116764265 2024-12-02T05:19:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,599 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 114fe259a63a4bf9b4e18dda95a67a82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733116764904 2024-12-02T05:19:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T05:19:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742355_1531 (size=31093) 2024-12-02T05:19:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,635 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#C#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,636 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/fcb9e1b2549d4c8ab010ba2632abd3b5 is 50, key is test_row_0/C:col10/1733116764913/Put/seqid=0 2024-12-02T05:19:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-02T05:19:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:26,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:26,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-02T05:19:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-02T05:19:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-02T05:19:26,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-12-02T05:19:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 159 msec 2024-12-02T05:19:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742356_1532 (size=12173) 2024-12-02T05:19:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,671 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/fcb9e1b2549d4c8ab010ba2632abd3b5 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/fcb9e1b2549d4c8ab010ba2632abd3b5 2024-12-02T05:19:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,676 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/C of 1160304dbb0b6d1b48d2c66c029eb555 into fcb9e1b2549d4c8ab010ba2632abd3b5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:26,676 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:26,676 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/C, priority=11, startTime=1733116766167; duration=0sec 2024-12-02T05:19:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,676 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:26,676 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:C 2024-12-02T05:19:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T05:19:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,817 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,818 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-02T05:19:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T05:19:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,821 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,822 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T05:19:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T05:19:26,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:26,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:26,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:26,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-02T05:19:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-02T05:19:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-02T05:19:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-12-02T05:19:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 162 msec 2024-12-02T05:19:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,026 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/935c8a36d37048d2b1eace19ff640dae as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/935c8a36d37048d2b1eace19ff640dae 2024-12-02T05:19:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,032 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/A of 1160304dbb0b6d1b48d2c66c029eb555 into 935c8a36d37048d2b1eace19ff640dae(size=30.4 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:27,032 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,032 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/A, priority=12, startTime=1733116766167; duration=0sec 2024-12-02T05:19:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,032 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:27,032 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:A 2024-12-02T05:19:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:27,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T05:19:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:27,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:27,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:27,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:27,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:27,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e441ec9761cd442db52fb92c1cd6d2da_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116767070/Put/seqid=0 2024-12-02T05:19:27,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116767121, max timestamp=1733116767128Num uniques 859; 2024-12-02T05:19:27,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:19:27,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116767121, max timestamp=1733116767128Num uniques 859; 2024-12-02T05:19:27,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:19:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T05:19:27,138 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:27,138 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-02T05:19:27,141 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T05:19:27,142 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:27,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:27,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116827144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116827146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116827147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116827149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116827151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742357_1533 (size=21918) 2024-12-02T05:19:27,162 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,167 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e441ec9761cd442db52fb92c1cd6d2da_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e441ec9761cd442db52fb92c1cd6d2da_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:27,169 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/41acca6c03394ed599820aa7995a29f8, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:27,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/41acca6c03394ed599820aa7995a29f8 is 175, key is test_row_0/A:col10/1733116767070/Put/seqid=0 2024-12-02T05:19:27,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742358_1534 (size=65323) 2024-12-02T05:19:27,219 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/41acca6c03394ed599820aa7995a29f8 2024-12-02T05:19:27,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/cd8a069509f04fb4ad545379ffdbe70d is 50, key is test_row_0/B:col10/1733116767070/Put/seqid=0 2024-12-02T05:19:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T05:19:27,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116827258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116827258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116827258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116827259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116827259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742359_1535 (size=12001) 2024-12-02T05:19:27,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/cd8a069509f04fb4ad545379ffdbe70d 2024-12-02T05:19:27,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/81754f48dde14de9af2d0987c329d7b3 is 50, key is test_row_0/C:col10/1733116767070/Put/seqid=0 2024-12-02T05:19:27,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742360_1536 (size=12001) 2024-12-02T05:19:27,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-02T05:19:27,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:27,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:27,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:27,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:27,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:27,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/81754f48dde14de9af2d0987c329d7b3 2024-12-02T05:19:27,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/41acca6c03394ed599820aa7995a29f8 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/41acca6c03394ed599820aa7995a29f8 2024-12-02T05:19:27,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/41acca6c03394ed599820aa7995a29f8, entries=350, sequenceid=110, filesize=63.8 K 2024-12-02T05:19:27,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/cd8a069509f04fb4ad545379ffdbe70d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/cd8a069509f04fb4ad545379ffdbe70d 2024-12-02T05:19:27,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/cd8a069509f04fb4ad545379ffdbe70d, entries=150, sequenceid=110, filesize=11.7 K 2024-12-02T05:19:27,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/81754f48dde14de9af2d0987c329d7b3 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/81754f48dde14de9af2d0987c329d7b3 2024-12-02T05:19:27,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/81754f48dde14de9af2d0987c329d7b3, entries=150, sequenceid=110, filesize=11.7 K 2024-12-02T05:19:27,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for 1160304dbb0b6d1b48d2c66c029eb555 in 227ms, sequenceid=110, compaction requested=true 2024-12-02T05:19:27,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:27,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:27,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:27,329 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:27,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:19:27,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:27,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:19:27,329 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:27,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:27,329 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127371 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:27,329 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:27,329 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/A is initiating minor compaction (all files) 2024-12-02T05:19:27,329 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/B is initiating minor compaction (all files) 2024-12-02T05:19:27,330 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/B in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:27,330 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/A in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:27,330 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7c9c9b27866243d388533733dce121ce, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/08795028871847df9741d443e4a0fbd8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/cd8a069509f04fb4ad545379ffdbe70d] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=35.3 K 2024-12-02T05:19:27,330 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/935c8a36d37048d2b1eace19ff640dae, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/934cc065d8464b66a36006dbd09c789d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/41acca6c03394ed599820aa7995a29f8] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=124.4 K 2024-12-02T05:19:27,330 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:27,330 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/935c8a36d37048d2b1eace19ff640dae, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/934cc065d8464b66a36006dbd09c789d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/41acca6c03394ed599820aa7995a29f8] 2024-12-02T05:19:27,330 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c9c9b27866243d388533733dce121ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733116764265 2024-12-02T05:19:27,330 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 935c8a36d37048d2b1eace19ff640dae, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733116764265 2024-12-02T05:19:27,330 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 934cc065d8464b66a36006dbd09c789d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733116764904 2024-12-02T05:19:27,331 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 08795028871847df9741d443e4a0fbd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733116764904 2024-12-02T05:19:27,331 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41acca6c03394ed599820aa7995a29f8, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733116767070 2024-12-02T05:19:27,331 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting cd8a069509f04fb4ad545379ffdbe70d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733116767070 2024-12-02T05:19:27,344 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#B#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:27,345 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/1973f69b18214bd58cef8e8417eef790 is 50, key is test_row_0/B:col10/1733116767070/Put/seqid=0 2024-12-02T05:19:27,347 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:27,349 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120278828093f3c148929737190958920927_1160304dbb0b6d1b48d2c66c029eb555 store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:27,351 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120278828093f3c148929737190958920927_1160304dbb0b6d1b48d2c66c029eb555, store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:27,351 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120278828093f3c148929737190958920927_1160304dbb0b6d1b48d2c66c029eb555 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:27,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742362_1538 (size=4469) 2024-12-02T05:19:27,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742361_1537 (size=12241) 2024-12-02T05:19:27,358 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#A#compaction#458 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:27,359 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/0613177e3c014cba99630b6838e67450 is 175, key is test_row_0/A:col10/1733116767070/Put/seqid=0 2024-12-02T05:19:27,364 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/1973f69b18214bd58cef8e8417eef790 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/1973f69b18214bd58cef8e8417eef790 2024-12-02T05:19:27,370 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/B of 1160304dbb0b6d1b48d2c66c029eb555 into 1973f69b18214bd58cef8e8417eef790(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:27,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742363_1539 (size=31195) 2024-12-02T05:19:27,370 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:27,370 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/B, priority=13, startTime=1733116767329; duration=0sec 2024-12-02T05:19:27,370 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:27,370 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:B 2024-12-02T05:19:27,370 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T05:19:27,371 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T05:19:27,371 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T05:19:27,371 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. because compaction request was cancelled 2024-12-02T05:19:27,371 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:C 2024-12-02T05:19:27,380 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/0613177e3c014cba99630b6838e67450 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/0613177e3c014cba99630b6838e67450 2024-12-02T05:19:27,394 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/A of 1160304dbb0b6d1b48d2c66c029eb555 into 0613177e3c014cba99630b6838e67450(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:27,394 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:27,394 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/A, priority=13, startTime=1733116767328; duration=0sec 2024-12-02T05:19:27,394 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:27,394 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:A 2024-12-02T05:19:27,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-02T05:19:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:27,453 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=131.94 KB heapSize=383.31 KB 2024-12-02T05:19:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:27,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T05:19:27,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027b3ed00396e94036a059bfcaaad49415_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116767149/Put/seqid=0 2024-12-02T05:19:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:27,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:27,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116827475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116827475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116827480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116827486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742364_1540 (size=12304) 2024-12-02T05:19:27,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:27,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116827498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,502 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027b3ed00396e94036a059bfcaaad49415_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027b3ed00396e94036a059bfcaaad49415_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/830a53e7683e4aafaa1e216760d50110, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:27,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/830a53e7683e4aafaa1e216760d50110 is 175, key is test_row_0/A:col10/1733116767149/Put/seqid=0 2024-12-02T05:19:27,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742365_1541 (size=31105) 2024-12-02T05:19:27,542 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/830a53e7683e4aafaa1e216760d50110 2024-12-02T05:19:27,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/88dab17453de4354815fb4803d4e4c61 is 50, key is test_row_0/B:col10/1733116767149/Put/seqid=0 2024-12-02T05:19:27,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742366_1542 (size=12151) 2024-12-02T05:19:27,583 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/88dab17453de4354815fb4803d4e4c61 2024-12-02T05:19:27,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116827587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116827587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116827598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116827598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/c627459a57d54d62912b93c28a0d5c5f is 50, key is test_row_0/C:col10/1733116767149/Put/seqid=0 2024-12-02T05:19:27,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116827608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742367_1543 (size=12151) 2024-12-02T05:19:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T05:19:27,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116827798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116827798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116827808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116827808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:27,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116827818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,058 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/c627459a57d54d62912b93c28a0d5c5f 2024-12-02T05:19:28,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/830a53e7683e4aafaa1e216760d50110 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/830a53e7683e4aafaa1e216760d50110 2024-12-02T05:19:28,067 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/830a53e7683e4aafaa1e216760d50110, entries=150, sequenceid=144, filesize=30.4 K 2024-12-02T05:19:28,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/88dab17453de4354815fb4803d4e4c61 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/88dab17453de4354815fb4803d4e4c61 2024-12-02T05:19:28,073 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/88dab17453de4354815fb4803d4e4c61, entries=150, sequenceid=144, filesize=11.9 K 2024-12-02T05:19:28,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/c627459a57d54d62912b93c28a0d5c5f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/c627459a57d54d62912b93c28a0d5c5f 2024-12-02T05:19:28,078 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/c627459a57d54d62912b93c28a0d5c5f, entries=150, sequenceid=144, filesize=11.9 K 2024-12-02T05:19:28,079 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~383.27 KB/392464, currentSize=53.67 KB/54960 for 1160304dbb0b6d1b48d2c66c029eb555 in 626ms, sequenceid=144, compaction requested=true 2024-12-02T05:19:28,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:28,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:28,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-02T05:19:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-02T05:19:28,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-02T05:19:28,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 940 msec 2024-12-02T05:19:28,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 946 msec 2024-12-02T05:19:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:28,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T05:19:28,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:28,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:28,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:28,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:28,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:28,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:28,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120264fc573d729b411bbb0cc707f469b732_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116767470/Put/seqid=0 2024-12-02T05:19:28,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116828144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116828144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116828145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116828147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116828150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742368_1544 (size=17284) 2024-12-02T05:19:28,188 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:28,193 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120264fc573d729b411bbb0cc707f469b732_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120264fc573d729b411bbb0cc707f469b732_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:28,194 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/7bea951cd0da4cda9ae8e2a757092298, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:28,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/7bea951cd0da4cda9ae8e2a757092298 is 175, key is test_row_0/A:col10/1733116767470/Put/seqid=0 2024-12-02T05:19:28,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742369_1545 (size=48389) 2024-12-02T05:19:28,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116828258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116828258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116828258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116828258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116828267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T05:19:28,278 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:28,279 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-02T05:19:28,284 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:28,285 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:28,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T05:19:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T05:19:28,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T05:19:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:28,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:28,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:28,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116828468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116828468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116828469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116828469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116828478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T05:19:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:28,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:28,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T05:19:28,618 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/7bea951cd0da4cda9ae8e2a757092298 2024-12-02T05:19:28,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/2761bcb40e984e4ab81fa5d2e6b5c469 is 50, key is test_row_0/B:col10/1733116767470/Put/seqid=0 2024-12-02T05:19:28,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742370_1546 (size=12151) 2024-12-02T05:19:28,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/2761bcb40e984e4ab81fa5d2e6b5c469 2024-12-02T05:19:28,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/8aee27890db54567b0f1ed30bba41220 is 50, key is test_row_0/C:col10/1733116767470/Put/seqid=0 2024-12-02T05:19:28,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T05:19:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:28,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:28,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742371_1547 (size=12151) 2024-12-02T05:19:28,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/8aee27890db54567b0f1ed30bba41220 2024-12-02T05:19:28,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/7bea951cd0da4cda9ae8e2a757092298 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/7bea951cd0da4cda9ae8e2a757092298 2024-12-02T05:19:28,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/7bea951cd0da4cda9ae8e2a757092298, entries=250, sequenceid=157, filesize=47.3 K 2024-12-02T05:19:28,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/2761bcb40e984e4ab81fa5d2e6b5c469 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/2761bcb40e984e4ab81fa5d2e6b5c469 2024-12-02T05:19:28,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116828781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116828781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/2761bcb40e984e4ab81fa5d2e6b5c469, entries=150, sequenceid=157, filesize=11.9 K 2024-12-02T05:19:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116828781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116828781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/8aee27890db54567b0f1ed30bba41220 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/8aee27890db54567b0f1ed30bba41220 2024-12-02T05:19:28,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116828787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:28,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/8aee27890db54567b0f1ed30bba41220, entries=150, sequenceid=157, filesize=11.9 K 2024-12-02T05:19:28,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1160304dbb0b6d1b48d2c66c029eb555 in 684ms, sequenceid=157, compaction requested=true 2024-12-02T05:19:28,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:28,797 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:28,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:28,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:28,798 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:28,799 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110689 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:28,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:19:28,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:28,799 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/A is initiating minor compaction (all files) 2024-12-02T05:19:28,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:19:28,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:28,799 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/A in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:28,800 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/0613177e3c014cba99630b6838e67450, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/830a53e7683e4aafaa1e216760d50110, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/7bea951cd0da4cda9ae8e2a757092298] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=108.1 K 2024-12-02T05:19:28,800 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:28,800 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/0613177e3c014cba99630b6838e67450, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/830a53e7683e4aafaa1e216760d50110, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/7bea951cd0da4cda9ae8e2a757092298] 2024-12-02T05:19:28,801 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0613177e3c014cba99630b6838e67450, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733116767070 2024-12-02T05:19:28,801 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:28,801 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/B is initiating minor compaction (all files) 2024-12-02T05:19:28,801 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/B in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:28,802 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/1973f69b18214bd58cef8e8417eef790, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/88dab17453de4354815fb4803d4e4c61, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/2761bcb40e984e4ab81fa5d2e6b5c469] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=35.7 K 2024-12-02T05:19:28,804 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 1973f69b18214bd58cef8e8417eef790, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733116767070 2024-12-02T05:19:28,804 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 830a53e7683e4aafaa1e216760d50110, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733116767127 2024-12-02T05:19:28,805 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 88dab17453de4354815fb4803d4e4c61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733116767127 2024-12-02T05:19:28,805 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bea951cd0da4cda9ae8e2a757092298, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733116767470 2024-12-02T05:19:28,806 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 2761bcb40e984e4ab81fa5d2e6b5c469, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733116767470 2024-12-02T05:19:28,826 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:28,827 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/c32c2c76282947a8b4627a0f633555aa is 50, key is test_row_0/B:col10/1733116767470/Put/seqid=0 2024-12-02T05:19:28,833 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:28,850 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120295fbee439ec94bf2acd99281d21cb305_1160304dbb0b6d1b48d2c66c029eb555 store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:28,852 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120295fbee439ec94bf2acd99281d21cb305_1160304dbb0b6d1b48d2c66c029eb555, store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:28,852 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120295fbee439ec94bf2acd99281d21cb305_1160304dbb0b6d1b48d2c66c029eb555 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:28,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742372_1548 (size=12493) 2024-12-02T05:19:28,881 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/c32c2c76282947a8b4627a0f633555aa as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/c32c2c76282947a8b4627a0f633555aa 2024-12-02T05:19:28,888 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/B of 1160304dbb0b6d1b48d2c66c029eb555 into c32c2c76282947a8b4627a0f633555aa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:28,888 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:28,888 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/B, priority=13, startTime=1733116768797; duration=0sec 2024-12-02T05:19:28,889 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:28,889 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:B 2024-12-02T05:19:28,889 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:19:28,890 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:19:28,891 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/C is initiating minor compaction (all files) 2024-12-02T05:19:28,891 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/C in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:28,891 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/fcb9e1b2549d4c8ab010ba2632abd3b5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/81754f48dde14de9af2d0987c329d7b3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/c627459a57d54d62912b93c28a0d5c5f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/8aee27890db54567b0f1ed30bba41220] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=47.3 K 2024-12-02T05:19:28,893 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting fcb9e1b2549d4c8ab010ba2632abd3b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733116764904 2024-12-02T05:19:28,893 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 81754f48dde14de9af2d0987c329d7b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733116767070 2024-12-02T05:19:28,894 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting c627459a57d54d62912b93c28a0d5c5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733116767127 2024-12-02T05:19:28,894 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aee27890db54567b0f1ed30bba41220, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733116767470 2024-12-02T05:19:28,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T05:19:28,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:28,899 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T05:19:28,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:28,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:28,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:28,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:28,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:28,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:28,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742373_1549 (size=4469) 2024-12-02T05:19:28,912 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#A#compaction#466 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:28,912 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/b49cd5ef90364719a26d8ed2119fb9b9 is 175, key is test_row_0/A:col10/1733116767470/Put/seqid=0 2024-12-02T05:19:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T05:19:28,943 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#C#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:28,943 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/c7e787446ea04d5da7628a69a444ac5a is 50, key is test_row_0/C:col10/1733116767470/Put/seqid=0 2024-12-02T05:19:29,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742374_1550 (size=31447) 2024-12-02T05:19:29,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b1ff3c308ed4473a8beb9c9a1c8aeffc_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116768143/Put/seqid=0 2024-12-02T05:19:29,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742375_1551 (size=12459) 2024-12-02T05:19:29,008 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/b49cd5ef90364719a26d8ed2119fb9b9 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b49cd5ef90364719a26d8ed2119fb9b9 2024-12-02T05:19:29,014 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/A of 1160304dbb0b6d1b48d2c66c029eb555 into b49cd5ef90364719a26d8ed2119fb9b9(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:29,014 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:29,014 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/A, priority=13, startTime=1733116768797; duration=0sec 2024-12-02T05:19:29,014 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:29,014 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:A 2024-12-02T05:19:29,024 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/c7e787446ea04d5da7628a69a444ac5a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/c7e787446ea04d5da7628a69a444ac5a 2024-12-02T05:19:29,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742376_1552 (size=12304) 2024-12-02T05:19:29,035 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/C of 1160304dbb0b6d1b48d2c66c029eb555 into c7e787446ea04d5da7628a69a444ac5a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:29,035 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:29,035 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/C, priority=12, startTime=1733116768799; duration=0sec 2024-12-02T05:19:29,036 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:29,036 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:C 2024-12-02T05:19:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:29,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:29,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116829309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116829310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116829314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116829316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116829316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116829418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T05:19:29,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116829428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116829428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116829429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116829429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,434 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b1ff3c308ed4473a8beb9c9a1c8aeffc_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b1ff3c308ed4473a8beb9c9a1c8aeffc_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:29,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/18ef469f5ad040c8b90018c4a415b63a, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:29,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/18ef469f5ad040c8b90018c4a415b63a is 175, key is test_row_0/A:col10/1733116768143/Put/seqid=0 2024-12-02T05:19:29,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742377_1553 (size=31105) 2024-12-02T05:19:29,478 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/18ef469f5ad040c8b90018c4a415b63a 2024-12-02T05:19:29,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/7321e2348e7045489418281e259af02e is 50, key is test_row_0/B:col10/1733116768143/Put/seqid=0 2024-12-02T05:19:29,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742378_1554 (size=12151) 2024-12-02T05:19:29,539 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/7321e2348e7045489418281e259af02e 2024-12-02T05:19:29,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/f591c18107044ca8a0987e8fb181d287 is 50, key is test_row_0/C:col10/1733116768143/Put/seqid=0 2024-12-02T05:19:29,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742379_1555 (size=12151) 2024-12-02T05:19:29,598 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/f591c18107044ca8a0987e8fb181d287 2024-12-02T05:19:29,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/18ef469f5ad040c8b90018c4a415b63a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/18ef469f5ad040c8b90018c4a415b63a 2024-12-02T05:19:29,610 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/18ef469f5ad040c8b90018c4a415b63a, entries=150, sequenceid=177, filesize=30.4 K 2024-12-02T05:19:29,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/7321e2348e7045489418281e259af02e as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7321e2348e7045489418281e259af02e 2024-12-02T05:19:29,615 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7321e2348e7045489418281e259af02e, entries=150, sequenceid=177, filesize=11.9 K 2024-12-02T05:19:29,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/f591c18107044ca8a0987e8fb181d287 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/f591c18107044ca8a0987e8fb181d287 2024-12-02T05:19:29,620 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/f591c18107044ca8a0987e8fb181d287, entries=150, sequenceid=177, filesize=11.9 K 2024-12-02T05:19:29,620 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 1160304dbb0b6d1b48d2c66c029eb555 in 722ms, sequenceid=177, compaction requested=false 2024-12-02T05:19:29,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:29,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:29,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-02T05:19:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-02T05:19:29,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-02T05:19:29,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3360 sec 2024-12-02T05:19:29,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.3460 sec 2024-12-02T05:19:29,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T05:19:29,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:29,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:29,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:29,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:29,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:29,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:29,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026ae0266fdecc473586d4e740f892e732_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116769313/Put/seqid=0 2024-12-02T05:19:29,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742380_1556 (size=12304) 2024-12-02T05:19:29,657 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:29,661 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026ae0266fdecc473586d4e740f892e732_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026ae0266fdecc473586d4e740f892e732_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:29,662 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/cec8d6a9c8af4509a5778d60b89cba25, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:29,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/cec8d6a9c8af4509a5778d60b89cba25 is 175, key is test_row_0/A:col10/1733116769313/Put/seqid=0 2024-12-02T05:19:29,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116829658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116829658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116829658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116829659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116829670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742381_1557 (size=31105) 2024-12-02T05:19:29,678 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/cec8d6a9c8af4509a5778d60b89cba25 2024-12-02T05:19:29,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/9822e00a2aa74fb49dbf81b990cce012 is 50, key is test_row_0/B:col10/1733116769313/Put/seqid=0 2024-12-02T05:19:29,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742382_1558 (size=12151) 2024-12-02T05:19:29,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/9822e00a2aa74fb49dbf81b990cce012 2024-12-02T05:19:29,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/a7ad578bce524aad9d98d20cdfbba35d is 50, key is test_row_0/C:col10/1733116769313/Put/seqid=0 2024-12-02T05:19:29,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742383_1559 (size=12151) 2024-12-02T05:19:29,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116829768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116829768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116829768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116829768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116829777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116829979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116829979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116829979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116829980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:29,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116829988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/a7ad578bce524aad9d98d20cdfbba35d 2024-12-02T05:19:30,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/cec8d6a9c8af4509a5778d60b89cba25 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/cec8d6a9c8af4509a5778d60b89cba25 2024-12-02T05:19:30,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/cec8d6a9c8af4509a5778d60b89cba25, entries=150, sequenceid=194, filesize=30.4 K 2024-12-02T05:19:30,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/9822e00a2aa74fb49dbf81b990cce012 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/9822e00a2aa74fb49dbf81b990cce012 2024-12-02T05:19:30,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/9822e00a2aa74fb49dbf81b990cce012, entries=150, sequenceid=194, filesize=11.9 K 2024-12-02T05:19:30,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/a7ad578bce524aad9d98d20cdfbba35d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/a7ad578bce524aad9d98d20cdfbba35d 2024-12-02T05:19:30,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/a7ad578bce524aad9d98d20cdfbba35d, entries=150, sequenceid=194, filesize=11.9 K 2024-12-02T05:19:30,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1160304dbb0b6d1b48d2c66c029eb555 in 516ms, sequenceid=194, compaction requested=true 2024-12-02T05:19:30,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:30,148 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:30,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:30,149 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:30,149 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/A is initiating minor compaction (all files) 2024-12-02T05:19:30,149 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/A in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:30,149 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b49cd5ef90364719a26d8ed2119fb9b9, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/18ef469f5ad040c8b90018c4a415b63a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/cec8d6a9c8af4509a5778d60b89cba25] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=91.5 K 2024-12-02T05:19:30,149 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:30,149 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b49cd5ef90364719a26d8ed2119fb9b9, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/18ef469f5ad040c8b90018c4a415b63a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/cec8d6a9c8af4509a5778d60b89cba25] 2024-12-02T05:19:30,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:30,150 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:30,151 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting b49cd5ef90364719a26d8ed2119fb9b9, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733116767470 2024-12-02T05:19:30,151 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18ef469f5ad040c8b90018c4a415b63a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733116768141 2024-12-02T05:19:30,151 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:30,152 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/B is initiating minor compaction (all files) 2024-12-02T05:19:30,152 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/B in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:30,152 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/c32c2c76282947a8b4627a0f633555aa, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7321e2348e7045489418281e259af02e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/9822e00a2aa74fb49dbf81b990cce012] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=35.9 K 2024-12-02T05:19:30,152 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting cec8d6a9c8af4509a5778d60b89cba25, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733116769308 2024-12-02T05:19:30,152 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting c32c2c76282947a8b4627a0f633555aa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733116767470 2024-12-02T05:19:30,153 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 7321e2348e7045489418281e259af02e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733116768141 2024-12-02T05:19:30,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:19:30,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:30,154 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 9822e00a2aa74fb49dbf81b990cce012, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733116769308 2024-12-02T05:19:30,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:19:30,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:30,165 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:30,169 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#B#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:30,170 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/eb56943e5cf44bd1ada4ce840f2fb60a is 50, key is test_row_0/B:col10/1733116769313/Put/seqid=0 2024-12-02T05:19:30,183 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202b60f84038a9c43668532f310985dbec0_1160304dbb0b6d1b48d2c66c029eb555 store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:30,185 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202b60f84038a9c43668532f310985dbec0_1160304dbb0b6d1b48d2c66c029eb555, store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:30,185 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b60f84038a9c43668532f310985dbec0_1160304dbb0b6d1b48d2c66c029eb555 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:30,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742384_1560 (size=12595) 2024-12-02T05:19:30,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742385_1561 (size=4469) 2024-12-02T05:19:30,231 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#A#compaction#474 average throughput is 0.37 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:30,232 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/1c649eca0baf4a5fab3c4e077e3b5c3b is 175, key is test_row_0/A:col10/1733116769313/Put/seqid=0 2024-12-02T05:19:30,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742386_1562 (size=31549) 2024-12-02T05:19:30,264 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/1c649eca0baf4a5fab3c4e077e3b5c3b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/1c649eca0baf4a5fab3c4e077e3b5c3b 2024-12-02T05:19:30,271 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/A of 1160304dbb0b6d1b48d2c66c029eb555 into 1c649eca0baf4a5fab3c4e077e3b5c3b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:30,271 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:30,271 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/A, priority=13, startTime=1733116770148; duration=0sec 2024-12-02T05:19:30,271 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:30,271 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:A 2024-12-02T05:19:30,271 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:30,272 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:30,272 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/C is initiating minor compaction (all files) 2024-12-02T05:19:30,272 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/C in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:30,272 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/c7e787446ea04d5da7628a69a444ac5a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/f591c18107044ca8a0987e8fb181d287, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/a7ad578bce524aad9d98d20cdfbba35d] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=35.9 K 2024-12-02T05:19:30,273 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7e787446ea04d5da7628a69a444ac5a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733116767470 2024-12-02T05:19:30,273 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting f591c18107044ca8a0987e8fb181d287, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733116768141 2024-12-02T05:19:30,273 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7ad578bce524aad9d98d20cdfbba35d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733116769308 2024-12-02T05:19:30,282 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:30,283 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/a1aaf142da744478bab73e8b07a802a6 is 50, key is test_row_0/C:col10/1733116769313/Put/seqid=0 2024-12-02T05:19:30,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742387_1563 (size=12561) 2024-12-02T05:19:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:30,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T05:19:30,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:30,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:30,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:30,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:30,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:30,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:30,295 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/a1aaf142da744478bab73e8b07a802a6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/a1aaf142da744478bab73e8b07a802a6 2024-12-02T05:19:30,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202578ea2b7a4854c408bf2f8e6cb685403_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116769652/Put/seqid=0 2024-12-02T05:19:30,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742388_1564 (size=12304) 2024-12-02T05:19:30,301 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:30,304 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202578ea2b7a4854c408bf2f8e6cb685403_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202578ea2b7a4854c408bf2f8e6cb685403_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:30,304 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/8afeb72180cb4fd8a7b1d7bda56b9718, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:30,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/8afeb72180cb4fd8a7b1d7bda56b9718 is 175, key is test_row_0/A:col10/1733116769652/Put/seqid=0 2024-12-02T05:19:30,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742389_1565 (size=31105) 2024-12-02T05:19:30,308 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/C of 1160304dbb0b6d1b48d2c66c029eb555 into a1aaf142da744478bab73e8b07a802a6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:30,308 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:30,308 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/C, priority=13, startTime=1733116770153; duration=0sec 2024-12-02T05:19:30,308 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:30,308 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:C 2024-12-02T05:19:30,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116830309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116830309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116830333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116830333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116830333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T05:19:30,438 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:30,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,438 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116830438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-02T05:19:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T05:19:30,440 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:30,440 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:30,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:30,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116830438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116830447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116830447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116830447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T05:19:30,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T05:19:30,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:30,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:30,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:30,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:30,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:30,601 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/eb56943e5cf44bd1ada4ce840f2fb60a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/eb56943e5cf44bd1ada4ce840f2fb60a 2024-12-02T05:19:30,604 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/B of 1160304dbb0b6d1b48d2c66c029eb555 into eb56943e5cf44bd1ada4ce840f2fb60a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:30,604 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:30,604 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/B, priority=13, startTime=1733116770150; duration=0sec 2024-12-02T05:19:30,604 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:30,604 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:B 2024-12-02T05:19:30,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116830647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116830647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116830657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116830657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116830657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,708 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/8afeb72180cb4fd8a7b1d7bda56b9718 2024-12-02T05:19:30,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/f5fbb76101044ca4a1a2938b13baf04a is 50, key is test_row_0/B:col10/1733116769652/Put/seqid=0 2024-12-02T05:19:30,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742390_1566 (size=12151) 2024-12-02T05:19:30,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T05:19:30,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:30,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:30,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:30,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:30,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T05:19:30,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T05:19:30,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:30,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:30,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:30,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:30,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:30,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116830958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116830958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116830967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116830967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:30,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116830967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:31,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T05:19:31,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:31,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:31,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:31,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:31,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T05:19:31,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/f5fbb76101044ca4a1a2938b13baf04a 2024-12-02T05:19:31,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/03b15fe2ccbe48cba4a721f07e12e712 is 50, key is test_row_0/C:col10/1733116769652/Put/seqid=0 2024-12-02T05:19:31,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742391_1567 (size=12151) 2024-12-02T05:19:31,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/03b15fe2ccbe48cba4a721f07e12e712 2024-12-02T05:19:31,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/8afeb72180cb4fd8a7b1d7bda56b9718 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/8afeb72180cb4fd8a7b1d7bda56b9718 2024-12-02T05:19:31,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/8afeb72180cb4fd8a7b1d7bda56b9718, entries=150, sequenceid=214, filesize=30.4 K 2024-12-02T05:19:31,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/f5fbb76101044ca4a1a2938b13baf04a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/f5fbb76101044ca4a1a2938b13baf04a 2024-12-02T05:19:31,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/f5fbb76101044ca4a1a2938b13baf04a, entries=150, sequenceid=214, filesize=11.9 K 2024-12-02T05:19:31,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/03b15fe2ccbe48cba4a721f07e12e712 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/03b15fe2ccbe48cba4a721f07e12e712 2024-12-02T05:19:31,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/03b15fe2ccbe48cba4a721f07e12e712, entries=150, sequenceid=214, filesize=11.9 K 2024-12-02T05:19:31,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=80.51 KB/82440 for 1160304dbb0b6d1b48d2c66c029eb555 in 859ms, sequenceid=214, compaction requested=false 2024-12-02T05:19:31,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:31,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T05:19:31,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:31,201 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T05:19:31,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:31,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:31,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:31,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:31,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:31,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:31,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bea67f6b416f4df4a38bcb6e7a04506d_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116770309/Put/seqid=0 2024-12-02T05:19:31,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742392_1568 (size=12304) 2024-12-02T05:19:31,279 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T05:19:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:31,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:31,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116831484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:31,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116831485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:31,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116831487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:31,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116831488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116831489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T05:19:31,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116831597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:31,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116831597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:31,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116831597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:31,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:31,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116831597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116831597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:31,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:31,614 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bea67f6b416f4df4a38bcb6e7a04506d_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bea67f6b416f4df4a38bcb6e7a04506d_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:31,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/06a10d599965409f88b5686182b1e647, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:31,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/06a10d599965409f88b5686182b1e647 is 175, key is test_row_0/A:col10/1733116770309/Put/seqid=0 2024-12-02T05:19:31,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742393_1569 (size=31105) 2024-12-02T05:19:31,632 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/06a10d599965409f88b5686182b1e647 2024-12-02T05:19:31,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/588c926a1faa4cce85a469d036ba8c2a is 50, key is test_row_0/B:col10/1733116770309/Put/seqid=0 2024-12-02T05:19:31,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742394_1570 (size=12151) 2024-12-02T05:19:31,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116831808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:31,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116831808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116831808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:31,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116831808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:31,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116831809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,076 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/588c926a1faa4cce85a469d036ba8c2a 2024-12-02T05:19:32,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/6183f36f1a3449469cda86fe7160cefe is 50, key is test_row_0/C:col10/1733116770309/Put/seqid=0 2024-12-02T05:19:32,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742395_1571 (size=12151) 2024-12-02T05:19:32,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116832117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116832128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116832128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116832128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116832128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,486 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/6183f36f1a3449469cda86fe7160cefe 2024-12-02T05:19:32,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/06a10d599965409f88b5686182b1e647 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/06a10d599965409f88b5686182b1e647 2024-12-02T05:19:32,492 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/06a10d599965409f88b5686182b1e647, entries=150, sequenceid=231, filesize=30.4 K 2024-12-02T05:19:32,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/588c926a1faa4cce85a469d036ba8c2a as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/588c926a1faa4cce85a469d036ba8c2a 2024-12-02T05:19:32,495 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/588c926a1faa4cce85a469d036ba8c2a, entries=150, sequenceid=231, filesize=11.9 K 2024-12-02T05:19:32,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/6183f36f1a3449469cda86fe7160cefe as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6183f36f1a3449469cda86fe7160cefe 2024-12-02T05:19:32,499 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6183f36f1a3449469cda86fe7160cefe, entries=150, sequenceid=231, filesize=11.9 K 2024-12-02T05:19:32,499 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1160304dbb0b6d1b48d2c66c029eb555 in 1298ms, sequenceid=231, compaction requested=true 2024-12-02T05:19:32,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:32,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:32,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-02T05:19:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-02T05:19:32,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-02T05:19:32,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0610 sec 2024-12-02T05:19:32,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.0650 sec 2024-12-02T05:19:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T05:19:32,588 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:32,588 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:32,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-02T05:19:32,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T05:19:32,590 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:32,590 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:32,590 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:32,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T05:19:32,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:32,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:32,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:32,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:32,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:32,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:32,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120265b1a5deff5f4b979bb7681abbd678ff_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116771486/Put/seqid=0 2024-12-02T05:19:32,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742396_1572 (size=12304) 2024-12-02T05:19:32,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116832644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116832647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116832647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116832648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116832649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T05:19:32,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T05:19:32,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:32,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:32,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:32,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:32,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:32,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116832758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116832758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116832758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116832758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116832758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T05:19:32,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:32,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:32,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:32,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:32,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T05:19:32,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116832968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116832968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116832968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116832968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116832968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:33,046 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,049 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120265b1a5deff5f4b979bb7681abbd678ff_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120265b1a5deff5f4b979bb7681abbd678ff_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:33,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T05:19:33,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:33,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,050 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/77aae4b2950141b391c3d3e2eb1303a0, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:33,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/77aae4b2950141b391c3d3e2eb1303a0 is 175, key is test_row_0/A:col10/1733116771486/Put/seqid=0 2024-12-02T05:19:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742397_1573 (size=31105) 2024-12-02T05:19:33,062 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/77aae4b2950141b391c3d3e2eb1303a0 2024-12-02T05:19:33,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/d6caf47ebb5a4be9832fd3c40eeb033d is 50, key is test_row_0/B:col10/1733116771486/Put/seqid=0 2024-12-02T05:19:33,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742398_1574 (size=12151) 2024-12-02T05:19:33,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T05:19:33,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:33,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T05:19:33,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:33,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116833278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116833278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:33,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:33,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116833278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116833278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:33,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:33,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116833278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:33,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T05:19:33,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:33,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/d6caf47ebb5a4be9832fd3c40eeb033d 2024-12-02T05:19:33,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/6e7ef333f0d547589baa9b78403e5842 is 50, key is test_row_0/C:col10/1733116771486/Put/seqid=0 2024-12-02T05:19:33,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742399_1575 (size=12151) 2024-12-02T05:19:33,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T05:19:33,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:33,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T05:19:33,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:33,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T05:19:33,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116833787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:33,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116833787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116833788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:33,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116833788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116833788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:33,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T05:19:33,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:33,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T05:19:33,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/6e7ef333f0d547589baa9b78403e5842 2024-12-02T05:19:33,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/77aae4b2950141b391c3d3e2eb1303a0 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/77aae4b2950141b391c3d3e2eb1303a0 2024-12-02T05:19:33,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/77aae4b2950141b391c3d3e2eb1303a0, entries=150, sequenceid=249, filesize=30.4 K 2024-12-02T05:19:33,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/d6caf47ebb5a4be9832fd3c40eeb033d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/d6caf47ebb5a4be9832fd3c40eeb033d 2024-12-02T05:19:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/d6caf47ebb5a4be9832fd3c40eeb033d, entries=150, sequenceid=249, filesize=11.9 K 2024-12-02T05:19:33,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/6e7ef333f0d547589baa9b78403e5842 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6e7ef333f0d547589baa9b78403e5842 2024-12-02T05:19:33,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6e7ef333f0d547589baa9b78403e5842, entries=150, sequenceid=249, filesize=11.9 K 2024-12-02T05:19:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1160304dbb0b6d1b48d2c66c029eb555 in 1279ms, sequenceid=249, compaction requested=true 2024-12-02T05:19:33,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:33,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:33,910 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:19:33,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:33,910 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:19:33,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:19:33,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:33,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:19:33,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:33,911 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124864 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:19:33,911 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:19:33,911 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/A is initiating minor compaction (all files) 2024-12-02T05:19:33,911 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/B is initiating minor compaction (all files) 2024-12-02T05:19:33,911 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/A in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,911 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/B in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,911 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/1c649eca0baf4a5fab3c4e077e3b5c3b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/8afeb72180cb4fd8a7b1d7bda56b9718, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/06a10d599965409f88b5686182b1e647, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/77aae4b2950141b391c3d3e2eb1303a0] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=121.9 K 2024-12-02T05:19:33,911 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/eb56943e5cf44bd1ada4ce840f2fb60a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/f5fbb76101044ca4a1a2938b13baf04a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/588c926a1faa4cce85a469d036ba8c2a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/d6caf47ebb5a4be9832fd3c40eeb033d] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=47.9 K 2024-12-02T05:19:33,911 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,911 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/1c649eca0baf4a5fab3c4e077e3b5c3b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/8afeb72180cb4fd8a7b1d7bda56b9718, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/06a10d599965409f88b5686182b1e647, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/77aae4b2950141b391c3d3e2eb1303a0] 2024-12-02T05:19:33,912 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting eb56943e5cf44bd1ada4ce840f2fb60a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733116769308 2024-12-02T05:19:33,912 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c649eca0baf4a5fab3c4e077e3b5c3b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733116769308 2024-12-02T05:19:33,912 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting f5fbb76101044ca4a1a2938b13baf04a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733116769652 2024-12-02T05:19:33,912 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8afeb72180cb4fd8a7b1d7bda56b9718, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733116769652 2024-12-02T05:19:33,913 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 588c926a1faa4cce85a469d036ba8c2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733116770309 2024-12-02T05:19:33,913 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06a10d599965409f88b5686182b1e647, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733116770309 2024-12-02T05:19:33,914 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting d6caf47ebb5a4be9832fd3c40eeb033d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733116771486 2024-12-02T05:19:33,914 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77aae4b2950141b391c3d3e2eb1303a0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733116771486 2024-12-02T05:19:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,932 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,933 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/fe0b57ccf6934bfaa68f6850c74e8234 is 50, key is test_row_0/B:col10/1733116771486/Put/seqid=0 2024-12-02T05:19:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,949 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,963 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202354ba4f633964a5db5fa276f33536cb0_1160304dbb0b6d1b48d2c66c029eb555 store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:33,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,965 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202354ba4f633964a5db5fa276f33536cb0_1160304dbb0b6d1b48d2c66c029eb555, store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:33,965 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202354ba4f633964a5db5fa276f33536cb0_1160304dbb0b6d1b48d2c66c029eb555 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T05:19:33,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:33,969 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T05:19:33,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:33,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:33,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:33,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:33,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:33,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:33,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742400_1576 (size=12731) 2024-12-02T05:19:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ddb37092439a4ac6b83c5af7037b533a_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_1/A:col10/1733116772648/Put/seqid=0 2024-12-02T05:19:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742401_1577 (size=4469) 2024-12-02T05:19:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742402_1578 (size=9914) 2024-12-02T05:19:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,465 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/fe0b57ccf6934bfaa68f6850c74e8234 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/fe0b57ccf6934bfaa68f6850c74e8234 2024-12-02T05:19:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,466 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#A#compaction#487 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,466 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/bbe0df15e7834460a1fac7c146118eae is 175, key is test_row_0/A:col10/1733116771486/Put/seqid=0 2024-12-02T05:19:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,469 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/B of 1160304dbb0b6d1b48d2c66c029eb555 into fe0b57ccf6934bfaa68f6850c74e8234(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:34,469 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:34,469 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/B, priority=12, startTime=1733116773910; duration=0sec 2024-12-02T05:19:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,469 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:34,469 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:B 2024-12-02T05:19:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,469 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T05:19:34,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,471 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T05:19:34,471 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/C is initiating minor compaction (all files) 2024-12-02T05:19:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,471 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/C in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:34,471 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/a1aaf142da744478bab73e8b07a802a6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/03b15fe2ccbe48cba4a721f07e12e712, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6183f36f1a3449469cda86fe7160cefe, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6e7ef333f0d547589baa9b78403e5842] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=47.9 K 2024-12-02T05:19:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,471 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting a1aaf142da744478bab73e8b07a802a6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733116769308 2024-12-02T05:19:34,472 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 03b15fe2ccbe48cba4a721f07e12e712, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733116769652 2024-12-02T05:19:34,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,472 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 6183f36f1a3449469cda86fe7160cefe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733116770309 2024-12-02T05:19:34,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,472 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e7ef333f0d547589baa9b78403e5842, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733116771486 2024-12-02T05:19:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742403_1579 (size=31685) 2024-12-02T05:19:34,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,483 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#C#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:34,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,483 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/cf9d4accc5bc473d8b060fb13411a8f2 is 50, key is test_row_0/C:col10/1733116771486/Put/seqid=0 2024-12-02T05:19:34,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,485 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/bbe0df15e7834460a1fac7c146118eae as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/bbe0df15e7834460a1fac7c146118eae 2024-12-02T05:19:34,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,492 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/A of 1160304dbb0b6d1b48d2c66c029eb555 into bbe0df15e7834460a1fac7c146118eae(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:34,492 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:34,492 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/A, priority=12, startTime=1733116773910; duration=0sec 2024-12-02T05:19:34,493 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:34,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,493 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:A 2024-12-02T05:19:34,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,497 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ddb37092439a4ac6b83c5af7037b533a_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ddb37092439a4ac6b83c5af7037b533a_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:34,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/106441211a76438fb9be65cf6d717d34, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:34,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/106441211a76438fb9be65cf6d717d34 is 175, key is test_row_1/A:col10/1733116772648/Put/seqid=0 2024-12-02T05:19:34,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742404_1580 (size=12697) 2024-12-02T05:19:34,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742405_1581 (size=22561) 2024-12-02T05:19:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,518 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/106441211a76438fb9be65cf6d717d34 2024-12-02T05:19:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/7fa4781e64934bf0beee8c022f8ffd66 is 50, key is test_row_1/B:col10/1733116772648/Put/seqid=0 2024-12-02T05:19:34,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742406_1582 (size=9857) 2024-12-02T05:19:34,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T05:19:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:34,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116834823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:34,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:34,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116834825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116834826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116834826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:34,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116834827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:34,917 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/cf9d4accc5bc473d8b060fb13411a8f2 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/cf9d4accc5bc473d8b060fb13411a8f2 2024-12-02T05:19:34,921 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/C of 1160304dbb0b6d1b48d2c66c029eb555 into cf9d4accc5bc473d8b060fb13411a8f2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:34,921 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:34,921 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/C, priority=12, startTime=1733116773910; duration=0sec 2024-12-02T05:19:34,921 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:34,921 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:C 2024-12-02T05:19:34,931 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/7fa4781e64934bf0beee8c022f8ffd66 2024-12-02T05:19:34,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:34,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116834938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116834938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:34,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:34,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116834938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116834938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:34,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116834938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:34,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/7604b665b16041dfb3f0ab1dae77a403 is 50, key is test_row_1/C:col10/1733116772648/Put/seqid=0 2024-12-02T05:19:34,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742407_1583 (size=9857) 2024-12-02T05:19:35,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116835148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116835148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116835148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116835148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116835148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,358 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/7604b665b16041dfb3f0ab1dae77a403 2024-12-02T05:19:35,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/106441211a76438fb9be65cf6d717d34 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/106441211a76438fb9be65cf6d717d34 2024-12-02T05:19:35,363 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/106441211a76438fb9be65cf6d717d34, entries=100, sequenceid=263, filesize=22.0 K 2024-12-02T05:19:35,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/7fa4781e64934bf0beee8c022f8ffd66 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7fa4781e64934bf0beee8c022f8ffd66 2024-12-02T05:19:35,366 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7fa4781e64934bf0beee8c022f8ffd66, entries=100, sequenceid=263, filesize=9.6 K 2024-12-02T05:19:35,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/7604b665b16041dfb3f0ab1dae77a403 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/7604b665b16041dfb3f0ab1dae77a403 2024-12-02T05:19:35,369 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/7604b665b16041dfb3f0ab1dae77a403, entries=100, sequenceid=263, filesize=9.6 K 2024-12-02T05:19:35,370 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1160304dbb0b6d1b48d2c66c029eb555 in 1402ms, sequenceid=263, compaction requested=false 2024-12-02T05:19:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-02T05:19:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-02T05:19:35,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-02T05:19:35,373 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7810 sec 2024-12-02T05:19:35,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.7850 sec 2024-12-02T05:19:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:35,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T05:19:35,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:35,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:35,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:35,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:35,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:35,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:35,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202861b107c043a4c578f19bd7bf65912af_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116775459/Put/seqid=0 2024-12-02T05:19:35,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116835466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742408_1584 (size=12454) 2024-12-02T05:19:35,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116835468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116835468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116835469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116835469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116835578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116835578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116835578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116835578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116835578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116835788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116835788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116835788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116835788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116835788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:35,871 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:35,873 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202861b107c043a4c578f19bd7bf65912af_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202861b107c043a4c578f19bd7bf65912af_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:35,874 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/4ea70794202b4900a5c4632f488ce9a6, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:35,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/4ea70794202b4900a5c4632f488ce9a6 is 175, key is test_row_0/A:col10/1733116775459/Put/seqid=0 2024-12-02T05:19:35,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742409_1585 (size=31255) 2024-12-02T05:19:36,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116836097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:36,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116836098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:36,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116836098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:36,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116836098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:36,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116836098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:36,319 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/4ea70794202b4900a5c4632f488ce9a6 2024-12-02T05:19:36,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/8382d9ca122e457ba38e5ca6baae2fae is 50, key is test_row_0/B:col10/1733116775459/Put/seqid=0 2024-12-02T05:19:36,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742410_1586 (size=12301) 2024-12-02T05:19:36,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116836607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:36,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:36,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116836607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:36,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:36,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:36,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:36,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116836608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116836607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:36,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116836608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:36,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/8382d9ca122e457ba38e5ca6baae2fae 2024-12-02T05:19:36,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/ab87aa6dd7a345a5af91557c36dba1bb is 50, key is test_row_0/C:col10/1733116775459/Put/seqid=0 2024-12-02T05:19:36,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742411_1587 (size=12301) 2024-12-02T05:19:36,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/ab87aa6dd7a345a5af91557c36dba1bb 2024-12-02T05:19:36,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/4ea70794202b4900a5c4632f488ce9a6 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/4ea70794202b4900a5c4632f488ce9a6 2024-12-02T05:19:36,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/4ea70794202b4900a5c4632f488ce9a6, entries=150, sequenceid=287, filesize=30.5 K 2024-12-02T05:19:36,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/8382d9ca122e457ba38e5ca6baae2fae as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/8382d9ca122e457ba38e5ca6baae2fae 2024-12-02T05:19:36,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/8382d9ca122e457ba38e5ca6baae2fae, entries=150, sequenceid=287, filesize=12.0 K 2024-12-02T05:19:36,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/ab87aa6dd7a345a5af91557c36dba1bb as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/ab87aa6dd7a345a5af91557c36dba1bb 2024-12-02T05:19:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T05:19:36,758 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:36,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/ab87aa6dd7a345a5af91557c36dba1bb, entries=150, sequenceid=287, filesize=12.0 K 2024-12-02T05:19:36,759 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:36,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 1160304dbb0b6d1b48d2c66c029eb555 in 1300ms, sequenceid=287, compaction requested=true 2024-12-02T05:19:36,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:36,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:36,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:36,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:19:36,760 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:36,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:36,760 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:36,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:19:36,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:36,760 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:36,761 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/B is initiating minor compaction (all files) 2024-12-02T05:19:36,761 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/B in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:36,761 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/fe0b57ccf6934bfaa68f6850c74e8234, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7fa4781e64934bf0beee8c022f8ffd66, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/8382d9ca122e457ba38e5ca6baae2fae] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=34.1 K 2024-12-02T05:19:36,761 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting fe0b57ccf6934bfaa68f6850c74e8234, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733116771486 2024-12-02T05:19:36,762 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:36,762 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/A is initiating minor compaction (all files) 2024-12-02T05:19:36,762 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/A in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:36,762 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/bbe0df15e7834460a1fac7c146118eae, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/106441211a76438fb9be65cf6d717d34, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/4ea70794202b4900a5c4632f488ce9a6] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=83.5 K 2024-12-02T05:19:36,762 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:36,762 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/bbe0df15e7834460a1fac7c146118eae, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/106441211a76438fb9be65cf6d717d34, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/4ea70794202b4900a5c4632f488ce9a6] 2024-12-02T05:19:36,762 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa4781e64934bf0beee8c022f8ffd66, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733116772642 2024-12-02T05:19:36,763 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbe0df15e7834460a1fac7c146118eae, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733116771486 2024-12-02T05:19:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-12-02T05:19:36,763 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 106441211a76438fb9be65cf6d717d34, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733116772642 2024-12-02T05:19:36,763 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 8382d9ca122e457ba38e5ca6baae2fae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733116775458 2024-12-02T05:19:36,763 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ea70794202b4900a5c4632f488ce9a6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733116775458 2024-12-02T05:19:36,764 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T05:19:36,765 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:36,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:36,773 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:36,773 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:36,774 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/0d9e328e98b541a981708bb2fc1e85ab is 50, key is test_row_0/B:col10/1733116775459/Put/seqid=0 2024-12-02T05:19:36,776 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120291510168f81542739bef584eac5a8457_1160304dbb0b6d1b48d2c66c029eb555 store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:36,777 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120291510168f81542739bef584eac5a8457_1160304dbb0b6d1b48d2c66c029eb555, store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:36,777 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120291510168f81542739bef584eac5a8457_1160304dbb0b6d1b48d2c66c029eb555 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:36,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742413_1589 (size=4469) 2024-12-02T05:19:36,795 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#A#compaction#496 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:36,796 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/7da0abf5385845d08786dd27edfcbe0f is 175, key is test_row_0/A:col10/1733116775459/Put/seqid=0 2024-12-02T05:19:36,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742414_1590 (size=31937) 2024-12-02T05:19:36,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742412_1588 (size=12983) 2024-12-02T05:19:36,805 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/7da0abf5385845d08786dd27edfcbe0f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/7da0abf5385845d08786dd27edfcbe0f 2024-12-02T05:19:36,809 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/A of 1160304dbb0b6d1b48d2c66c029eb555 into 7da0abf5385845d08786dd27edfcbe0f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:36,809 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:36,809 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/A, priority=13, startTime=1733116776760; duration=0sec 2024-12-02T05:19:36,809 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:36,809 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:A 2024-12-02T05:19:36,809 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:36,810 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:36,811 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/C is initiating minor compaction (all files) 2024-12-02T05:19:36,811 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/C in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:36,811 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/cf9d4accc5bc473d8b060fb13411a8f2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/7604b665b16041dfb3f0ab1dae77a403, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/ab87aa6dd7a345a5af91557c36dba1bb] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=34.0 K 2024-12-02T05:19:36,811 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf9d4accc5bc473d8b060fb13411a8f2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733116771486 2024-12-02T05:19:36,812 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7604b665b16041dfb3f0ab1dae77a403, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733116772642 2024-12-02T05:19:36,812 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab87aa6dd7a345a5af91557c36dba1bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733116775458 2024-12-02T05:19:36,822 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#C#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:36,822 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/6cffd571f20f46279a23c352a491cf00 is 50, key is test_row_0/C:col10/1733116775459/Put/seqid=0 2024-12-02T05:19:36,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742415_1591 (size=12949) 2024-12-02T05:19:36,844 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/6cffd571f20f46279a23c352a491cf00 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6cffd571f20f46279a23c352a491cf00 2024-12-02T05:19:36,849 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/C of 1160304dbb0b6d1b48d2c66c029eb555 into 6cffd571f20f46279a23c352a491cf00(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:36,849 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:36,849 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/C, priority=13, startTime=1733116776760; duration=0sec 2024-12-02T05:19:36,849 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:36,849 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:C 2024-12-02T05:19:36,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T05:19:36,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T05:19:36,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:36,917 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T05:19:36,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:36,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:36,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:36,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:36,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d237bc9b8e884a359fbe985a16a1ae76_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116775467/Put/seqid=0 2024-12-02T05:19:36,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742416_1592 (size=12454) 2024-12-02T05:19:36,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:36,937 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d237bc9b8e884a359fbe985a16a1ae76_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d237bc9b8e884a359fbe985a16a1ae76_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:36,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/b059f005ed4e4b498e160c66ef2ba691, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:36,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/b059f005ed4e4b498e160c66ef2ba691 is 175, key is test_row_0/A:col10/1733116775467/Put/seqid=0 2024-12-02T05:19:36,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742417_1593 (size=31255) 2024-12-02T05:19:36,944 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/b059f005ed4e4b498e160c66ef2ba691 2024-12-02T05:19:36,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/b966a509122f45789de241e516ae583b is 50, key is test_row_0/B:col10/1733116775467/Put/seqid=0 2024-12-02T05:19:36,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742418_1594 (size=12301) 2024-12-02T05:19:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T05:19:37,204 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/0d9e328e98b541a981708bb2fc1e85ab as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/0d9e328e98b541a981708bb2fc1e85ab 2024-12-02T05:19:37,208 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/B of 1160304dbb0b6d1b48d2c66c029eb555 into 0d9e328e98b541a981708bb2fc1e85ab(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:37,208 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:37,208 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/B, priority=13, startTime=1733116776760; duration=0sec 2024-12-02T05:19:37,208 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:37,208 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:B 2024-12-02T05:19:37,355 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/b966a509122f45789de241e516ae583b 2024-12-02T05:19:37,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/47ad0420138a40d2b516200a61af8fb7 is 50, key is test_row_0/C:col10/1733116775467/Put/seqid=0 2024-12-02T05:19:37,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742419_1595 (size=12301) 2024-12-02T05:19:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T05:19:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:37,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:37,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116777626, max timestamp=1733116777632Num uniques 859; 2024-12-02T05:19:37,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:19:37,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116777626, max timestamp=1733116777632Num uniques 859; 2024-12-02T05:19:37,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:19:37,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733116777626, max timestamp=1733116777632Num uniques 859; 2024-12-02T05:19:37,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T05:19:37,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116837671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:37,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116837671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:37,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116837675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:37,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116837675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:37,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116837676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:37,773 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/47ad0420138a40d2b516200a61af8fb7 2024-12-02T05:19:37,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/b059f005ed4e4b498e160c66ef2ba691 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b059f005ed4e4b498e160c66ef2ba691 2024-12-02T05:19:37,779 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b059f005ed4e4b498e160c66ef2ba691, entries=150, sequenceid=300, filesize=30.5 K 2024-12-02T05:19:37,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/b966a509122f45789de241e516ae583b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/b966a509122f45789de241e516ae583b 2024-12-02T05:19:37,782 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/b966a509122f45789de241e516ae583b, entries=150, sequenceid=300, filesize=12.0 K 2024-12-02T05:19:37,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/47ad0420138a40d2b516200a61af8fb7 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/47ad0420138a40d2b516200a61af8fb7 2024-12-02T05:19:37,786 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/47ad0420138a40d2b516200a61af8fb7, entries=150, sequenceid=300, filesize=12.0 K 2024-12-02T05:19:37,787 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 1160304dbb0b6d1b48d2c66c029eb555 in 870ms, sequenceid=300, compaction requested=false 2024-12-02T05:19:37,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:37,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:37,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-12-02T05:19:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-12-02T05:19:37,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-02T05:19:37,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0230 sec 2024-12-02T05:19:37,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.0310 sec 2024-12-02T05:19:37,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733116777670, max timestamp=1733116777788Num uniques 1018; 2024-12-02T05:19:37,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f49461030c22:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T05:19:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:37,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=116.29 KB heapSize=332.05 KB 2024-12-02T05:19:37,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:37,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T05:19:37,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:37,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T05:19:37,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:37,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T05:19:37,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120233958411759f42d6912aa1b95806696c_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116777791/Put/seqid=0 2024-12-02T05:19:37,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:37,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116837805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116837804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:37,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116837805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:37,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116837813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:37,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116837813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:37,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742420_1596 (size=12454) 2024-12-02T05:19:37,823 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:37,828 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120233958411759f42d6912aa1b95806696c_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120233958411759f42d6912aa1b95806696c_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:37,829 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/384155b0bf1f48bc986b010e158f682d, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:37,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/384155b0bf1f48bc986b010e158f682d is 175, key is test_row_0/A:col10/1733116777791/Put/seqid=0 2024-12-02T05:19:37,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742421_1597 (size=31255) 2024-12-02T05:19:37,838 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=341, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/384155b0bf1f48bc986b010e158f682d 2024-12-02T05:19:37,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/206fd7e68bf346b9b2df622e5e8f9346 is 50, key is test_row_0/B:col10/1733116777791/Put/seqid=0 2024-12-02T05:19:37,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742422_1598 (size=12301) 2024-12-02T05:19:37,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/206fd7e68bf346b9b2df622e5e8f9346 2024-12-02T05:19:37,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/915682d33eb34482be9f626e5a98d318 is 50, key is test_row_0/C:col10/1733116777791/Put/seqid=0 2024-12-02T05:19:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T05:19:37,897 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:37,898 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-12-02T05:19:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T05:19:37,901 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:37,902 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:37,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:37,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116837918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:37,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116837918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116837919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:37,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:37,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116837928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116837928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:37,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:37,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:37,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:37,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742423_1599 (size=12301) 2024-12-02T05:19:37,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/915682d33eb34482be9f626e5a98d318 2024-12-02T05:19:37,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/384155b0bf1f48bc986b010e158f682d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/384155b0bf1f48bc986b010e158f682d 2024-12-02T05:19:37,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/384155b0bf1f48bc986b010e158f682d, entries=150, sequenceid=341, filesize=30.5 K 2024-12-02T05:19:37,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/206fd7e68bf346b9b2df622e5e8f9346 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/206fd7e68bf346b9b2df622e5e8f9346 2024-12-02T05:19:37,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/206fd7e68bf346b9b2df622e5e8f9346, entries=150, sequenceid=341, filesize=12.0 K 2024-12-02T05:19:37,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/915682d33eb34482be9f626e5a98d318 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/915682d33eb34482be9f626e5a98d318 2024-12-02T05:19:37,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/915682d33eb34482be9f626e5a98d318, entries=150, sequenceid=341, filesize=12.0 K 2024-12-02T05:19:37,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~116.29 KB/119080, heapSize ~332 KB/339968, currentSize=60.38 KB/61830 for 1160304dbb0b6d1b48d2c66c029eb555 in 172ms, sequenceid=341, compaction requested=true 2024-12-02T05:19:37,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:37,965 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T05:19:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T05:19:37,965 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:37,966 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:37,966 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:37,966 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/A is initiating minor compaction (all files) 2024-12-02T05:19:37,966 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/B is initiating minor compaction (all files) 2024-12-02T05:19:37,966 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/A in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:37,966 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/7da0abf5385845d08786dd27edfcbe0f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b059f005ed4e4b498e160c66ef2ba691, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/384155b0bf1f48bc986b010e158f682d] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=92.2 K 2024-12-02T05:19:37,966 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:37,966 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. files: [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/7da0abf5385845d08786dd27edfcbe0f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b059f005ed4e4b498e160c66ef2ba691, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/384155b0bf1f48bc986b010e158f682d] 2024-12-02T05:19:37,966 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/B in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:37,966 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/0d9e328e98b541a981708bb2fc1e85ab, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/b966a509122f45789de241e516ae583b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/206fd7e68bf346b9b2df622e5e8f9346] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=36.7 K 2024-12-02T05:19:37,967 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7da0abf5385845d08786dd27edfcbe0f, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733116775458 2024-12-02T05:19:37,967 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d9e328e98b541a981708bb2fc1e85ab, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733116775458 2024-12-02T05:19:37,967 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting b059f005ed4e4b498e160c66ef2ba691, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733116775467 2024-12-02T05:19:37,967 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting b966a509122f45789de241e516ae583b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733116775467 2024-12-02T05:19:37,968 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 206fd7e68bf346b9b2df622e5e8f9346, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733116777675 2024-12-02T05:19:37,968 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] compactions.Compactor(225): Compacting 384155b0bf1f48bc986b010e158f682d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733116777675 2024-12-02T05:19:37,980 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:37,998 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#B#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:37,998 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/3cba123f674744c6aaea7965d3399f3c is 50, key is test_row_0/B:col10/1733116777791/Put/seqid=0 2024-12-02T05:19:38,003 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412027ad15c762ea84824965a080004b100e1_1160304dbb0b6d1b48d2c66c029eb555 store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:38,004 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412027ad15c762ea84824965a080004b100e1_1160304dbb0b6d1b48d2c66c029eb555, store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:38,004 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027ad15c762ea84824965a080004b100e1_1160304dbb0b6d1b48d2c66c029eb555 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T05:19:38,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742424_1600 (size=13085) 2024-12-02T05:19:38,048 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/3cba123f674744c6aaea7965d3399f3c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/3cba123f674744c6aaea7965d3399f3c 2024-12-02T05:19:38,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T05:19:38,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:38,055 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T05:19:38,055 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/B of 1160304dbb0b6d1b48d2c66c029eb555 into 3cba123f674744c6aaea7965d3399f3c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:38,055 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:38,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:38,055 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/B, priority=13, startTime=1733116777965; duration=0sec 2024-12-02T05:19:38,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:38,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:38,055 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T05:19:38,055 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:B 2024-12-02T05:19:38,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:38,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:38,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:38,055 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T05:19:38,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742425_1601 (size=4469) 2024-12-02T05:19:38,057 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T05:19:38,057 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1541): 1160304dbb0b6d1b48d2c66c029eb555/C is initiating minor compaction (all files) 2024-12-02T05:19:38,057 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1160304dbb0b6d1b48d2c66c029eb555/C in TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:38,058 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6cffd571f20f46279a23c352a491cf00, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/47ad0420138a40d2b516200a61af8fb7, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/915682d33eb34482be9f626e5a98d318] into tmpdir=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp, totalSize=36.7 K 2024-12-02T05:19:38,058 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#A#compaction#504 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:38,058 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/6cd686db6bab4268a0930cbc5f14936b is 175, key is test_row_0/A:col10/1733116777791/Put/seqid=0 2024-12-02T05:19:38,060 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cffd571f20f46279a23c352a491cf00, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733116775458 2024-12-02T05:19:38,069 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 47ad0420138a40d2b516200a61af8fb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733116775467 2024-12-02T05:19:38,073 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] compactions.Compactor(225): Compacting 915682d33eb34482be9f626e5a98d318, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733116777675 2024-12-02T05:19:38,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021bc8c3b5a0b94856aa91b1c3a562fad1_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116777810/Put/seqid=0 2024-12-02T05:19:38,095 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1160304dbb0b6d1b48d2c66c029eb555#C#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T05:19:38,096 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/a13802388149490f92acab3975e08983 is 50, key is test_row_0/C:col10/1733116777791/Put/seqid=0 2024-12-02T05:19:38,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742426_1602 (size=32039) 2024-12-02T05:19:38,116 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/6cd686db6bab4268a0930cbc5f14936b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/6cd686db6bab4268a0930cbc5f14936b 2024-12-02T05:19:38,121 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/A of 1160304dbb0b6d1b48d2c66c029eb555 into 6cd686db6bab4268a0930cbc5f14936b(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:38,121 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:38,121 INFO [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/A, priority=13, startTime=1733116777964; duration=0sec 2024-12-02T05:19:38,121 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:38,121 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:A 2024-12-02T05:19:38,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742428_1604 (size=13051) 2024-12-02T05:19:38,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742427_1603 (size=12454) 2024-12-02T05:19:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:38,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:38,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116838177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116838179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116838179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116838182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116838193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T05:19:38,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116838288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116838288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116838297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116838298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116838297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116838497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116838497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116838507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116838507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116838517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T05:19:38,529 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/a13802388149490f92acab3975e08983 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/a13802388149490f92acab3975e08983 2024-12-02T05:19:38,529 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021bc8c3b5a0b94856aa91b1c3a562fad1_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021bc8c3b5a0b94856aa91b1c3a562fad1_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:38,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/b75558828dcb4254b25c7b48134b1fa9, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:38,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/b75558828dcb4254b25c7b48134b1fa9 is 175, key is test_row_0/A:col10/1733116777810/Put/seqid=0 2024-12-02T05:19:38,532 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1160304dbb0b6d1b48d2c66c029eb555/C of 1160304dbb0b6d1b48d2c66c029eb555 into a13802388149490f92acab3975e08983(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T05:19:38,532 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:38,532 INFO [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., storeName=1160304dbb0b6d1b48d2c66c029eb555/C, priority=13, startTime=1733116777965; duration=0sec 2024-12-02T05:19:38,532 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:38,532 DEBUG [RS:0;f49461030c22:46495-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:C 2024-12-02T05:19:38,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742429_1605 (size=31255) 2024-12-02T05:19:38,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116838807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116838807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116838818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116838818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116838827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:38,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:38,943 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/b75558828dcb4254b25c7b48134b1fa9 2024-12-02T05:19:38,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/11cc980c6fc44253837cfd3a8c66efa0 is 50, key is test_row_0/B:col10/1733116777810/Put/seqid=0 2024-12-02T05:19:38,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742430_1606 (size=12301) 2024-12-02T05:19:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T05:19:39,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116839317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:39,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116839318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:39,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:39,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:39,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:39,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116839327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:39,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:39,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:39,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:39,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116839327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:39,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:39,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:39,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:39,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116839337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:39,355 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/11cc980c6fc44253837cfd3a8c66efa0 2024-12-02T05:19:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/4b966157acf64f9e9b3d54ea3a822166 is 50, key is test_row_0/C:col10/1733116777810/Put/seqid=0 2024-12-02T05:19:39,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742431_1607 (size=12301) 2024-12-02T05:19:39,392 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/4b966157acf64f9e9b3d54ea3a822166 2024-12-02T05:19:39,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/b75558828dcb4254b25c7b48134b1fa9 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b75558828dcb4254b25c7b48134b1fa9 2024-12-02T05:19:39,398 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b75558828dcb4254b25c7b48134b1fa9, entries=150, sequenceid=354, filesize=30.5 K 2024-12-02T05:19:39,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/11cc980c6fc44253837cfd3a8c66efa0 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/11cc980c6fc44253837cfd3a8c66efa0 2024-12-02T05:19:39,403 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/11cc980c6fc44253837cfd3a8c66efa0, entries=150, sequenceid=354, filesize=12.0 K 2024-12-02T05:19:39,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/4b966157acf64f9e9b3d54ea3a822166 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/4b966157acf64f9e9b3d54ea3a822166 2024-12-02T05:19:39,408 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/4b966157acf64f9e9b3d54ea3a822166, entries=150, sequenceid=354, filesize=12.0 K 2024-12-02T05:19:39,408 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1160304dbb0b6d1b48d2c66c029eb555 in 1353ms, sequenceid=354, compaction requested=false 2024-12-02T05:19:39,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:39,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:39,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-12-02T05:19:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-12-02T05:19:39,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-02T05:19:39,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5080 sec 2024-12-02T05:19:39,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.5150 sec 2024-12-02T05:19:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T05:19:40,048 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:40,048 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T05:19:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-12-02T05:19:40,050 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T05:19:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T05:19:40,051 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T05:19:40,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T05:19:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T05:19:40,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-12-02T05:19:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:40,203 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T05:19:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:40,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028139fa7f0d5345baaf8850f40f1fcd4b_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116778174/Put/seqid=0 2024-12-02T05:19:40,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742432_1608 (size=12454) 2024-12-02T05:19:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:40,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. as already flushing 2024-12-02T05:19:40,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116840339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:40,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116840343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:40,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:40,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116840344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:40,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:40,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:40,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39304 deadline: 1733116840347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:40,350 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in 1160304dbb0b6d1b48d2c66c029eb555 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:19:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:40,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39386 deadline: 1733116840358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:40,360 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f49461030c22,46495,1733116594459 for 'test_row_2' in 1160304dbb0b6d1b48d2c66c029eb555 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T05:19:40,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T05:19:40,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116840450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:40,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116840457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:40,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:40,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116840457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:40,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:40,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:40,615 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028139fa7f0d5345baaf8850f40f1fcd4b_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028139fa7f0d5345baaf8850f40f1fcd4b_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:40,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/ba146346aed641a19601deb8d7d00634, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:40,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/ba146346aed641a19601deb8d7d00634 is 175, key is test_row_0/A:col10/1733116778174/Put/seqid=0 2024-12-02T05:19:40,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742433_1609 (size=31255) 2024-12-02T05:19:40,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116840657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:40,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116840667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116840667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T05:19:40,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39326 deadline: 1733116840967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:40,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:40,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39342 deadline: 1733116840977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:40,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T05:19:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39374 deadline: 1733116840977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 2024-12-02T05:19:40,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 , the old value is region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T05:19:40,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., hostname=f49461030c22,46495,1733116594459, seqNum=5 because the exception is null or not the one we care about 2024-12-02T05:19:41,026 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/ba146346aed641a19601deb8d7d00634 2024-12-02T05:19:41,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/7e0d65603de84909bb7a9954021bc96b is 50, key is test_row_0/B:col10/1733116778174/Put/seqid=0 2024-12-02T05:19:41,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742434_1610 (size=12301) 2024-12-02T05:19:41,035 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/7e0d65603de84909bb7a9954021bc96b 2024-12-02T05:19:41,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/f285217110094b748d002d2e1cf889d2 is 50, key is test_row_0/C:col10/1733116778174/Put/seqid=0 2024-12-02T05:19:41,057 INFO [Thread-2303 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2303. 2024-12-02T05:19:41,057 DEBUG [Thread-2303 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:41,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742435_1611 (size=12301) 2024-12-02T05:19:41,057 DEBUG [Thread-2303 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,057 DEBUG [Thread-2303 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,057 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:41,058 INFO [Thread-2297 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2297. 2024-12-02T05:19:41,058 DEBUG [Thread-2297 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:41,058 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,058 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,058 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:41,059 INFO [Thread-2299 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2299. 2024-12-02T05:19:41,059 DEBUG [Thread-2299 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:41,060 DEBUG [Thread-2299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,060 DEBUG [Thread-2299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:41,060 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/f285217110094b748d002d2e1cf889d2 2024-12-02T05:19:41,061 INFO [Thread-2301 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2301. 2024-12-02T05:19:41,061 DEBUG [Thread-2301 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:41,061 DEBUG [Thread-2301 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,061 DEBUG [Thread-2301 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:41,061 INFO [Thread-2305 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2305. 2024-12-02T05:19:41,062 DEBUG [Thread-2305 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:41,062 DEBUG [Thread-2305 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,062 DEBUG [Thread-2305 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,062 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:41,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/ba146346aed641a19601deb8d7d00634 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/ba146346aed641a19601deb8d7d00634 2024-12-02T05:19:41,067 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/ba146346aed641a19601deb8d7d00634, entries=150, sequenceid=376, filesize=30.5 K 2024-12-02T05:19:41,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/7e0d65603de84909bb7a9954021bc96b as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7e0d65603de84909bb7a9954021bc96b 2024-12-02T05:19:41,070 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7e0d65603de84909bb7a9954021bc96b, entries=150, sequenceid=376, filesize=12.0 K 2024-12-02T05:19:41,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/f285217110094b748d002d2e1cf889d2 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/f285217110094b748d002d2e1cf889d2 2024-12-02T05:19:41,074 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/f285217110094b748d002d2e1cf889d2, entries=150, sequenceid=376, filesize=12.0 K 2024-12-02T05:19:41,074 INFO [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1160304dbb0b6d1b48d2c66c029eb555 in 871ms, sequenceid=376, compaction requested=true 2024-12-02T05:19:41,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:41,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:41,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f49461030c22:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-12-02T05:19:41,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-12-02T05:19:41,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-12-02T05:19:41,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0250 sec 2024-12-02T05:19:41,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 1.0300 sec 2024-12-02T05:19:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T05:19:41,188 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46495 {}] regionserver.HRegion(8855): Flush requested on 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:41,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T05:19:41,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:41,479 INFO [Thread-2288 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2288. 2024-12-02T05:19:41,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:41,479 DEBUG [Thread-2288 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:41,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:41,479 DEBUG [Thread-2288 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:41,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:41,479 DEBUG [Thread-2288 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:41,479 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:41,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120224e6620876124871a65d81961479d9e5_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116780330/Put/seqid=0 2024-12-02T05:19:41,489 INFO [Thread-2286 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2286. 2024-12-02T05:19:41,489 INFO [Thread-2292 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2292. 2024-12-02T05:19:41,489 DEBUG [Thread-2292 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:41,489 DEBUG [Thread-2286 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:41,489 DEBUG [Thread-2292 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,489 DEBUG [Thread-2286 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742436_1612 (size=12454) 2024-12-02T05:19:41,489 DEBUG [Thread-2292 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,489 DEBUG [Thread-2286 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:41,490 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:41,490 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:41,890 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:41,895 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120224e6620876124871a65d81961479d9e5_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120224e6620876124871a65d81961479d9e5_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:41,897 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/c75795edc74f4458ba56825e338f1ab2, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:41,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/c75795edc74f4458ba56825e338f1ab2 is 175, key is test_row_0/A:col10/1733116780330/Put/seqid=0 2024-12-02T05:19:41,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742437_1613 (size=31255) 2024-12-02T05:19:42,303 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/c75795edc74f4458ba56825e338f1ab2 2024-12-02T05:19:42,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/a482696b51a5486db7eec966ee2fcb0f is 50, key is test_row_0/B:col10/1733116780330/Put/seqid=0 2024-12-02T05:19:42,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742438_1614 (size=12301) 2024-12-02T05:19:42,370 INFO [Thread-2294 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2294. 2024-12-02T05:19:42,370 DEBUG [Thread-2294 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:42,370 DEBUG [Thread-2294 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:42,370 DEBUG [Thread-2294 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:42,370 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:42,381 INFO [Thread-2290 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2290. 2024-12-02T05:19:42,381 DEBUG [Thread-2290 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T05:19:42,381 DEBUG [Thread-2290 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:42,381 DEBUG [Thread-2290 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:42,381 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:42,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T05:19:42,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-12-02T05:19:42,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-12-02T05:19:42,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-12-02T05:19:42,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-12-02T05:19:42,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-02T05:19:42,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T05:19:42,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3810 2024-12-02T05:19:42,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3756 2024-12-02T05:19:42,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3724 2024-12-02T05:19:42,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3823 2024-12-02T05:19:42,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3751 2024-12-02T05:19:42,382 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T05:19:42,382 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T05:19:42,382 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T05:19:42,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:42,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:42,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:42,384 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T05:19:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T05:19:42,389 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116782389"}]},"ts":"1733116782389"} 2024-12-02T05:19:42,391 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T05:19:42,392 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T05:19:42,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T05:19:42,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1160304dbb0b6d1b48d2c66c029eb555, UNASSIGN}] 2024-12-02T05:19:42,395 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1160304dbb0b6d1b48d2c66c029eb555, UNASSIGN 2024-12-02T05:19:42,396 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=176 updating hbase:meta row=1160304dbb0b6d1b48d2c66c029eb555, regionState=CLOSING, regionLocation=f49461030c22,46495,1733116594459 2024-12-02T05:19:42,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1160304dbb0b6d1b48d2c66c029eb555, UNASSIGN because future has completed 2024-12-02T05:19:42,398 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T05:19:42,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459}] 2024-12-02T05:19:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T05:19:42,551 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(122): Close 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:42,551 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T05:19:42,551 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1722): Closing 1160304dbb0b6d1b48d2c66c029eb555, disabling compactions & flushes 2024-12-02T05:19:42,551 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T05:19:42,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/a482696b51a5486db7eec966ee2fcb0f 2024-12-02T05:19:42,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/389ef19c52e34a79a99c21fbaf9b8f1c is 50, key is test_row_0/C:col10/1733116780330/Put/seqid=0 2024-12-02T05:19:42,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742439_1615 (size=12301) 2024-12-02T05:19:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T05:19:43,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/389ef19c52e34a79a99c21fbaf9b8f1c 2024-12-02T05:19:43,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/c75795edc74f4458ba56825e338f1ab2 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/c75795edc74f4458ba56825e338f1ab2 2024-12-02T05:19:43,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/c75795edc74f4458ba56825e338f1ab2, entries=150, sequenceid=389, filesize=30.5 K 2024-12-02T05:19:43,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/a482696b51a5486db7eec966ee2fcb0f as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/a482696b51a5486db7eec966ee2fcb0f 2024-12-02T05:19:43,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/a482696b51a5486db7eec966ee2fcb0f, entries=150, sequenceid=389, filesize=12.0 K 2024-12-02T05:19:43,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/389ef19c52e34a79a99c21fbaf9b8f1c as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/389ef19c52e34a79a99c21fbaf9b8f1c 2024-12-02T05:19:43,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/389ef19c52e34a79a99c21fbaf9b8f1c, entries=150, sequenceid=389, filesize=12.0 K 2024-12-02T05:19:43,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for 1160304dbb0b6d1b48d2c66c029eb555 in 1666ms, sequenceid=389, compaction requested=true 2024-12-02T05:19:43,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1160304dbb0b6d1b48d2c66c029eb555: 2024-12-02T05:19:43,145 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:43,145 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:43,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:43,145 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. after waiting 0 ms 2024-12-02T05:19:43,145 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. because compaction request was cancelled 2024-12-02T05:19:43,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:43,145 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:43,145 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:A 2024-12-02T05:19:43,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:B, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:43,145 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. because compaction request was cancelled 2024-12-02T05:19:43,145 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(2902): Flushing 1160304dbb0b6d1b48d2c66c029eb555 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-02T05:19:43,145 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:B 2024-12-02T05:19:43,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:43,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1160304dbb0b6d1b48d2c66c029eb555:C, priority=-2147483648, current under compaction store size is 1 2024-12-02T05:19:43,145 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. because compaction request was cancelled 2024-12-02T05:19:43,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T05:19:43,145 DEBUG [RS:0;f49461030c22:46495-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1160304dbb0b6d1b48d2c66c029eb555:C 2024-12-02T05:19:43,145 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=A 2024-12-02T05:19:43,145 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:43,145 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=B 2024-12-02T05:19:43,145 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:43,145 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1160304dbb0b6d1b48d2c66c029eb555, store=C 2024-12-02T05:19:43,145 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T05:19:43,151 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027f9061947d4d4d1790ef6c01fa9272dc_1160304dbb0b6d1b48d2c66c029eb555 is 50, key is test_row_0/A:col10/1733116781488/Put/seqid=0 2024-12-02T05:19:43,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742440_1616 (size=9914) 2024-12-02T05:19:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T05:19:43,554 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T05:19:43,557 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027f9061947d4d4d1790ef6c01fa9272dc_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027f9061947d4d4d1790ef6c01fa9272dc_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:43,558 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/5b3caeb4c7bb429c874c152a81fc3f3d, store: [table=TestAcidGuarantees family=A region=1160304dbb0b6d1b48d2c66c029eb555] 2024-12-02T05:19:43,558 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/5b3caeb4c7bb429c874c152a81fc3f3d is 175, key is test_row_0/A:col10/1733116781488/Put/seqid=0 2024-12-02T05:19:43,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742441_1617 (size=22561) 2024-12-02T05:19:43,962 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=396, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/5b3caeb4c7bb429c874c152a81fc3f3d 2024-12-02T05:19:43,967 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/a7d53c748ade47a6b45828be73308756 is 50, key is test_row_0/B:col10/1733116781488/Put/seqid=0 2024-12-02T05:19:43,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742442_1618 (size=9857) 2024-12-02T05:19:44,377 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/a7d53c748ade47a6b45828be73308756 2024-12-02T05:19:44,386 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/d59aebde800e4f06972b26c318058b55 is 50, key is test_row_0/C:col10/1733116781488/Put/seqid=0 2024-12-02T05:19:44,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742443_1619 (size=9857) 2024-12-02T05:19:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T05:19:44,791 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/d59aebde800e4f06972b26c318058b55 2024-12-02T05:19:44,798 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/A/5b3caeb4c7bb429c874c152a81fc3f3d as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/5b3caeb4c7bb429c874c152a81fc3f3d 2024-12-02T05:19:44,802 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/5b3caeb4c7bb429c874c152a81fc3f3d, entries=100, sequenceid=396, filesize=22.0 K 2024-12-02T05:19:44,803 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/B/a7d53c748ade47a6b45828be73308756 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/a7d53c748ade47a6b45828be73308756 2024-12-02T05:19:44,805 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/a7d53c748ade47a6b45828be73308756, entries=100, sequenceid=396, filesize=9.6 K 2024-12-02T05:19:44,806 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/.tmp/C/d59aebde800e4f06972b26c318058b55 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/d59aebde800e4f06972b26c318058b55 2024-12-02T05:19:44,808 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/d59aebde800e4f06972b26c318058b55, entries=100, sequenceid=396, filesize=9.6 K 2024-12-02T05:19:44,809 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1160304dbb0b6d1b48d2c66c029eb555 in 1664ms, sequenceid=396, compaction requested=true 2024-12-02T05:19:44,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/5327f319f2944748b41d27303a63e6c2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/85441ae48cc34a269a0100cae6e733f8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/a809c8c8a6514ce3bc9a691965ab72e0, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/640fbae191dd449da5b6066786812895, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/935c8a36d37048d2b1eace19ff640dae, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/934cc065d8464b66a36006dbd09c789d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/41acca6c03394ed599820aa7995a29f8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/0613177e3c014cba99630b6838e67450, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/830a53e7683e4aafaa1e216760d50110, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/7bea951cd0da4cda9ae8e2a757092298, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b49cd5ef90364719a26d8ed2119fb9b9, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/18ef469f5ad040c8b90018c4a415b63a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/1c649eca0baf4a5fab3c4e077e3b5c3b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/cec8d6a9c8af4509a5778d60b89cba25, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/8afeb72180cb4fd8a7b1d7bda56b9718, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/06a10d599965409f88b5686182b1e647, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/bbe0df15e7834460a1fac7c146118eae, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/77aae4b2950141b391c3d3e2eb1303a0, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/106441211a76438fb9be65cf6d717d34, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/7da0abf5385845d08786dd27edfcbe0f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/4ea70794202b4900a5c4632f488ce9a6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b059f005ed4e4b498e160c66ef2ba691, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/384155b0bf1f48bc986b010e158f682d] to archive 2024-12-02T05:19:44,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:19:44,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/5327f319f2944748b41d27303a63e6c2 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/5327f319f2944748b41d27303a63e6c2 2024-12-02T05:19:44,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/85441ae48cc34a269a0100cae6e733f8 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/85441ae48cc34a269a0100cae6e733f8 2024-12-02T05:19:44,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/a809c8c8a6514ce3bc9a691965ab72e0 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/a809c8c8a6514ce3bc9a691965ab72e0 2024-12-02T05:19:44,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/640fbae191dd449da5b6066786812895 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/640fbae191dd449da5b6066786812895 2024-12-02T05:19:44,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/935c8a36d37048d2b1eace19ff640dae to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/935c8a36d37048d2b1eace19ff640dae 2024-12-02T05:19:44,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/934cc065d8464b66a36006dbd09c789d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/934cc065d8464b66a36006dbd09c789d 2024-12-02T05:19:44,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/41acca6c03394ed599820aa7995a29f8 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/41acca6c03394ed599820aa7995a29f8 2024-12-02T05:19:44,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/0613177e3c014cba99630b6838e67450 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/0613177e3c014cba99630b6838e67450 2024-12-02T05:19:44,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/830a53e7683e4aafaa1e216760d50110 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/830a53e7683e4aafaa1e216760d50110 2024-12-02T05:19:44,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/7bea951cd0da4cda9ae8e2a757092298 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/7bea951cd0da4cda9ae8e2a757092298 2024-12-02T05:19:44,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b49cd5ef90364719a26d8ed2119fb9b9 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b49cd5ef90364719a26d8ed2119fb9b9 2024-12-02T05:19:44,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/18ef469f5ad040c8b90018c4a415b63a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/18ef469f5ad040c8b90018c4a415b63a 2024-12-02T05:19:44,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/1c649eca0baf4a5fab3c4e077e3b5c3b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/1c649eca0baf4a5fab3c4e077e3b5c3b 2024-12-02T05:19:44,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/cec8d6a9c8af4509a5778d60b89cba25 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/cec8d6a9c8af4509a5778d60b89cba25 2024-12-02T05:19:44,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/8afeb72180cb4fd8a7b1d7bda56b9718 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/8afeb72180cb4fd8a7b1d7bda56b9718 2024-12-02T05:19:44,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/06a10d599965409f88b5686182b1e647 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/06a10d599965409f88b5686182b1e647 2024-12-02T05:19:44,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/bbe0df15e7834460a1fac7c146118eae to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/bbe0df15e7834460a1fac7c146118eae 2024-12-02T05:19:44,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/77aae4b2950141b391c3d3e2eb1303a0 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/77aae4b2950141b391c3d3e2eb1303a0 2024-12-02T05:19:44,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/106441211a76438fb9be65cf6d717d34 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/106441211a76438fb9be65cf6d717d34 2024-12-02T05:19:44,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/7da0abf5385845d08786dd27edfcbe0f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/7da0abf5385845d08786dd27edfcbe0f 2024-12-02T05:19:44,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/4ea70794202b4900a5c4632f488ce9a6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/4ea70794202b4900a5c4632f488ce9a6 2024-12-02T05:19:44,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b059f005ed4e4b498e160c66ef2ba691 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b059f005ed4e4b498e160c66ef2ba691 2024-12-02T05:19:44,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/384155b0bf1f48bc986b010e158f682d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/384155b0bf1f48bc986b010e158f682d 2024-12-02T05:19:44,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/96c8c1079b6941c697200942cff2098e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/381f6649ff2b40e09b82b31132c081fd, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/36b14806b9ed47f7a7582d1779d0b99b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7c9c9b27866243d388533733dce121ce, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/e63354599f3e4dc28a50bcdc8b9f4edf, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/08795028871847df9741d443e4a0fbd8, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/1973f69b18214bd58cef8e8417eef790, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/cd8a069509f04fb4ad545379ffdbe70d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/88dab17453de4354815fb4803d4e4c61, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/c32c2c76282947a8b4627a0f633555aa, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/2761bcb40e984e4ab81fa5d2e6b5c469, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7321e2348e7045489418281e259af02e, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/eb56943e5cf44bd1ada4ce840f2fb60a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/9822e00a2aa74fb49dbf81b990cce012, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/f5fbb76101044ca4a1a2938b13baf04a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/588c926a1faa4cce85a469d036ba8c2a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/fe0b57ccf6934bfaa68f6850c74e8234, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/d6caf47ebb5a4be9832fd3c40eeb033d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7fa4781e64934bf0beee8c022f8ffd66, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/0d9e328e98b541a981708bb2fc1e85ab, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/8382d9ca122e457ba38e5ca6baae2fae, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/b966a509122f45789de241e516ae583b, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/206fd7e68bf346b9b2df622e5e8f9346] to archive 2024-12-02T05:19:44,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:19:44,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/96c8c1079b6941c697200942cff2098e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/96c8c1079b6941c697200942cff2098e 2024-12-02T05:19:44,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/381f6649ff2b40e09b82b31132c081fd to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/381f6649ff2b40e09b82b31132c081fd 2024-12-02T05:19:44,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/36b14806b9ed47f7a7582d1779d0b99b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/36b14806b9ed47f7a7582d1779d0b99b 2024-12-02T05:19:44,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7c9c9b27866243d388533733dce121ce to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7c9c9b27866243d388533733dce121ce 2024-12-02T05:19:44,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/e63354599f3e4dc28a50bcdc8b9f4edf to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/e63354599f3e4dc28a50bcdc8b9f4edf 2024-12-02T05:19:44,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/08795028871847df9741d443e4a0fbd8 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/08795028871847df9741d443e4a0fbd8 2024-12-02T05:19:44,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/1973f69b18214bd58cef8e8417eef790 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/1973f69b18214bd58cef8e8417eef790 2024-12-02T05:19:44,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/cd8a069509f04fb4ad545379ffdbe70d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/cd8a069509f04fb4ad545379ffdbe70d 2024-12-02T05:19:44,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/88dab17453de4354815fb4803d4e4c61 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/88dab17453de4354815fb4803d4e4c61 2024-12-02T05:19:44,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/c32c2c76282947a8b4627a0f633555aa to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/c32c2c76282947a8b4627a0f633555aa 2024-12-02T05:19:44,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/2761bcb40e984e4ab81fa5d2e6b5c469 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/2761bcb40e984e4ab81fa5d2e6b5c469 2024-12-02T05:19:44,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7321e2348e7045489418281e259af02e to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7321e2348e7045489418281e259af02e 2024-12-02T05:19:44,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/eb56943e5cf44bd1ada4ce840f2fb60a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/eb56943e5cf44bd1ada4ce840f2fb60a 2024-12-02T05:19:44,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/9822e00a2aa74fb49dbf81b990cce012 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/9822e00a2aa74fb49dbf81b990cce012 2024-12-02T05:19:44,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/f5fbb76101044ca4a1a2938b13baf04a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/f5fbb76101044ca4a1a2938b13baf04a 2024-12-02T05:19:44,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/588c926a1faa4cce85a469d036ba8c2a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/588c926a1faa4cce85a469d036ba8c2a 2024-12-02T05:19:44,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/fe0b57ccf6934bfaa68f6850c74e8234 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/fe0b57ccf6934bfaa68f6850c74e8234 2024-12-02T05:19:44,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/d6caf47ebb5a4be9832fd3c40eeb033d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/d6caf47ebb5a4be9832fd3c40eeb033d 2024-12-02T05:19:44,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7fa4781e64934bf0beee8c022f8ffd66 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7fa4781e64934bf0beee8c022f8ffd66 2024-12-02T05:19:44,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/0d9e328e98b541a981708bb2fc1e85ab to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/0d9e328e98b541a981708bb2fc1e85ab 2024-12-02T05:19:44,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/8382d9ca122e457ba38e5ca6baae2fae to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/8382d9ca122e457ba38e5ca6baae2fae 2024-12-02T05:19:44,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/b966a509122f45789de241e516ae583b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/b966a509122f45789de241e516ae583b 2024-12-02T05:19:44,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/206fd7e68bf346b9b2df622e5e8f9346 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/206fd7e68bf346b9b2df622e5e8f9346 2024-12-02T05:19:44,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/03033c5059774f4fab5cdeccbfc16e53, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6b2ba22a5aa040658584aa97d0f52095, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/b7d33be043f547ce8b70719e2d523a2f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/97a032671ade4dfc9801515ff17e8e31, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/fcb9e1b2549d4c8ab010ba2632abd3b5, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/114fe259a63a4bf9b4e18dda95a67a82, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/81754f48dde14de9af2d0987c329d7b3, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/c627459a57d54d62912b93c28a0d5c5f, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/c7e787446ea04d5da7628a69a444ac5a, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/8aee27890db54567b0f1ed30bba41220, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/f591c18107044ca8a0987e8fb181d287, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/a1aaf142da744478bab73e8b07a802a6, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/a7ad578bce524aad9d98d20cdfbba35d, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/03b15fe2ccbe48cba4a721f07e12e712, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6183f36f1a3449469cda86fe7160cefe, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/cf9d4accc5bc473d8b060fb13411a8f2, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6e7ef333f0d547589baa9b78403e5842, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/7604b665b16041dfb3f0ab1dae77a403, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6cffd571f20f46279a23c352a491cf00, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/ab87aa6dd7a345a5af91557c36dba1bb, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/47ad0420138a40d2b516200a61af8fb7, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/915682d33eb34482be9f626e5a98d318] to archive 2024-12-02T05:19:44,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T05:19:44,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/03033c5059774f4fab5cdeccbfc16e53 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/03033c5059774f4fab5cdeccbfc16e53 2024-12-02T05:19:44,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6b2ba22a5aa040658584aa97d0f52095 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6b2ba22a5aa040658584aa97d0f52095 2024-12-02T05:19:44,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/b7d33be043f547ce8b70719e2d523a2f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/b7d33be043f547ce8b70719e2d523a2f 2024-12-02T05:19:44,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/97a032671ade4dfc9801515ff17e8e31 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/97a032671ade4dfc9801515ff17e8e31 2024-12-02T05:19:44,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/fcb9e1b2549d4c8ab010ba2632abd3b5 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/fcb9e1b2549d4c8ab010ba2632abd3b5 2024-12-02T05:19:44,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/114fe259a63a4bf9b4e18dda95a67a82 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/114fe259a63a4bf9b4e18dda95a67a82 2024-12-02T05:19:44,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/81754f48dde14de9af2d0987c329d7b3 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/81754f48dde14de9af2d0987c329d7b3 2024-12-02T05:19:44,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/c627459a57d54d62912b93c28a0d5c5f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/c627459a57d54d62912b93c28a0d5c5f 2024-12-02T05:19:44,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/c7e787446ea04d5da7628a69a444ac5a to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/c7e787446ea04d5da7628a69a444ac5a 2024-12-02T05:19:44,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/8aee27890db54567b0f1ed30bba41220 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/8aee27890db54567b0f1ed30bba41220 2024-12-02T05:19:44,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/f591c18107044ca8a0987e8fb181d287 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/f591c18107044ca8a0987e8fb181d287 2024-12-02T05:19:44,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/a1aaf142da744478bab73e8b07a802a6 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/a1aaf142da744478bab73e8b07a802a6 2024-12-02T05:19:44,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/a7ad578bce524aad9d98d20cdfbba35d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/a7ad578bce524aad9d98d20cdfbba35d 2024-12-02T05:19:44,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/03b15fe2ccbe48cba4a721f07e12e712 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/03b15fe2ccbe48cba4a721f07e12e712 2024-12-02T05:19:44,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6183f36f1a3449469cda86fe7160cefe to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6183f36f1a3449469cda86fe7160cefe 2024-12-02T05:19:44,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/cf9d4accc5bc473d8b060fb13411a8f2 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/cf9d4accc5bc473d8b060fb13411a8f2 2024-12-02T05:19:44,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6e7ef333f0d547589baa9b78403e5842 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6e7ef333f0d547589baa9b78403e5842 2024-12-02T05:19:44,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/7604b665b16041dfb3f0ab1dae77a403 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/7604b665b16041dfb3f0ab1dae77a403 2024-12-02T05:19:44,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6cffd571f20f46279a23c352a491cf00 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/6cffd571f20f46279a23c352a491cf00 2024-12-02T05:19:44,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/ab87aa6dd7a345a5af91557c36dba1bb to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/ab87aa6dd7a345a5af91557c36dba1bb 2024-12-02T05:19:44,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/47ad0420138a40d2b516200a61af8fb7 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/47ad0420138a40d2b516200a61af8fb7 2024-12-02T05:19:44,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/915682d33eb34482be9f626e5a98d318 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/915682d33eb34482be9f626e5a98d318 2024-12-02T05:19:44,896 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/recovered.edits/399.seqid, newMaxSeqId=399, maxSeqId=4 2024-12-02T05:19:44,897 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. 2024-12-02T05:19:44,897 DEBUG [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1676): Region close journal for 1160304dbb0b6d1b48d2c66c029eb555: Waiting for close lock at 1733116782551Running coprocessor pre-close hooks at 1733116782551Disabling compacts and flushes for region at 1733116782551Disabling writes for close at 1733116783145 (+594 ms)Obtaining lock to block concurrent updates at 1733116783145Preparing flush snapshotting stores in 1160304dbb0b6d1b48d2c66c029eb555 at 1733116783145Finished memstore snapshotting TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733116783145Flushing stores of TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555. at 1733116783146 (+1 ms)Flushing 1160304dbb0b6d1b48d2c66c029eb555/A: creating writer at 1733116783146Flushing mob file 1160304dbb0b6d1b48d2c66c029eb555/A: appending metadata at 1733116783151 (+5 ms)Flushing mob file 1160304dbb0b6d1b48d2c66c029eb555/A: closing flushed file at 1733116783151Flushing 1160304dbb0b6d1b48d2c66c029eb555/A: appending metadata at 1733116783558 (+407 ms)Flushing 1160304dbb0b6d1b48d2c66c029eb555/A: closing flushed file at 1733116783558Flushing 1160304dbb0b6d1b48d2c66c029eb555/B: creating writer at 1733116783965 (+407 ms)Flushing 1160304dbb0b6d1b48d2c66c029eb555/B: appending metadata at 1733116783967 (+2 ms)Flushing 1160304dbb0b6d1b48d2c66c029eb555/B: closing flushed file at 1733116783967Flushing 1160304dbb0b6d1b48d2c66c029eb555/C: creating writer at 1733116784381 (+414 ms)Flushing 1160304dbb0b6d1b48d2c66c029eb555/C: appending metadata at 1733116784385 (+4 ms)Flushing 1160304dbb0b6d1b48d2c66c029eb555/C: closing flushed file at 1733116784385Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f965cbc: reopening flushed file at 1733116784798 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ce8ccc5: reopening flushed file at 1733116784802 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d12abc7: reopening flushed file at 1733116784805 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1160304dbb0b6d1b48d2c66c029eb555 in 1664ms, sequenceid=396, compaction requested=true at 1733116784809 (+4 ms)Writing region close event to WAL at 1733116784894 (+85 ms)Running coprocessor post-close hooks at 1733116784897 (+3 ms)Closed at 1733116784897 2024-12-02T05:19:44,899 INFO [RS_CLOSE_REGION-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(157): Closed 1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:44,899 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=176 updating hbase:meta row=1160304dbb0b6d1b48d2c66c029eb555, regionState=CLOSED 2024-12-02T05:19:44,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=177, ppid=176, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 because future has completed 2024-12-02T05:19:44,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-12-02T05:19:44,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; CloseRegionProcedure 1160304dbb0b6d1b48d2c66c029eb555, server=f49461030c22,46495,1733116594459 in 2.5030 sec 2024-12-02T05:19:44,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=176, resume processing ppid=175 2024-12-02T05:19:44,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, ppid=175, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1160304dbb0b6d1b48d2c66c029eb555, UNASSIGN in 2.5090 sec 2024-12-02T05:19:44,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-12-02T05:19:44,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5140 sec 2024-12-02T05:19:44,909 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733116784908"}]},"ts":"1733116784908"} 2024-12-02T05:19:44,910 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T05:19:44,910 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T05:19:44,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5260 sec 2024-12-02T05:19:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T05:19:46,549 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:46,550 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T05:19:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:46,555 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=178, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-02T05:19:46,556 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=178, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:46,560 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,563 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C, FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/recovered.edits] 2024-12-02T05:19:46,566 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/5b3caeb4c7bb429c874c152a81fc3f3d to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/5b3caeb4c7bb429c874c152a81fc3f3d 2024-12-02T05:19:46,567 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/6cd686db6bab4268a0930cbc5f14936b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/6cd686db6bab4268a0930cbc5f14936b 2024-12-02T05:19:46,569 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b75558828dcb4254b25c7b48134b1fa9 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/b75558828dcb4254b25c7b48134b1fa9 2024-12-02T05:19:46,570 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/ba146346aed641a19601deb8d7d00634 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/ba146346aed641a19601deb8d7d00634 2024-12-02T05:19:46,572 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/c75795edc74f4458ba56825e338f1ab2 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/A/c75795edc74f4458ba56825e338f1ab2 2024-12-02T05:19:46,575 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/11cc980c6fc44253837cfd3a8c66efa0 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/11cc980c6fc44253837cfd3a8c66efa0 2024-12-02T05:19:46,577 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/3cba123f674744c6aaea7965d3399f3c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/3cba123f674744c6aaea7965d3399f3c 2024-12-02T05:19:46,578 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7e0d65603de84909bb7a9954021bc96b to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/7e0d65603de84909bb7a9954021bc96b 2024-12-02T05:19:46,580 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/a482696b51a5486db7eec966ee2fcb0f to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/a482696b51a5486db7eec966ee2fcb0f 2024-12-02T05:19:46,582 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/a7d53c748ade47a6b45828be73308756 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/B/a7d53c748ade47a6b45828be73308756 2024-12-02T05:19:46,585 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/389ef19c52e34a79a99c21fbaf9b8f1c to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/389ef19c52e34a79a99c21fbaf9b8f1c 2024-12-02T05:19:46,586 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/4b966157acf64f9e9b3d54ea3a822166 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/4b966157acf64f9e9b3d54ea3a822166 2024-12-02T05:19:46,588 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/a13802388149490f92acab3975e08983 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/a13802388149490f92acab3975e08983 2024-12-02T05:19:46,590 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/d59aebde800e4f06972b26c318058b55 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/d59aebde800e4f06972b26c318058b55 2024-12-02T05:19:46,591 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/f285217110094b748d002d2e1cf889d2 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/C/f285217110094b748d002d2e1cf889d2 2024-12-02T05:19:46,594 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/recovered.edits/399.seqid to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555/recovered.edits/399.seqid 2024-12-02T05:19:46,595 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/default/TestAcidGuarantees/1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,595 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T05:19:46,596 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T05:19:46,596 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T05:19:46,598 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020c25218d40b04533a8a30c0041d64d33_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020c25218d40b04533a8a30c0041d64d33_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,598 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021bc8c3b5a0b94856aa91b1c3a562fad1_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021bc8c3b5a0b94856aa91b1c3a562fad1_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,599 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120224e6620876124871a65d81961479d9e5_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120224e6620876124871a65d81961479d9e5_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,600 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120233958411759f42d6912aa1b95806696c_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120233958411759f42d6912aa1b95806696c_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,600 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120247bb747234db4044b1d8a685a372e217_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120247bb747234db4044b1d8a685a372e217_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,601 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202558341c09fd84582925ac73de2fe8946_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202558341c09fd84582925ac73de2fe8946_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,602 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202578ea2b7a4854c408bf2f8e6cb685403_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202578ea2b7a4854c408bf2f8e6cb685403_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,602 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202593b6bce48b344308bfdcc9b2f5f5ab6_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202593b6bce48b344308bfdcc9b2f5f5ab6_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,603 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120264fc573d729b411bbb0cc707f469b732_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120264fc573d729b411bbb0cc707f469b732_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,604 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120265b1a5deff5f4b979bb7681abbd678ff_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120265b1a5deff5f4b979bb7681abbd678ff_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,604 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026ae0266fdecc473586d4e740f892e732_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026ae0266fdecc473586d4e740f892e732_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,605 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027b3ed00396e94036a059bfcaaad49415_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027b3ed00396e94036a059bfcaaad49415_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,606 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027f9061947d4d4d1790ef6c01fa9272dc_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027f9061947d4d4d1790ef6c01fa9272dc_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,606 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028139fa7f0d5345baaf8850f40f1fcd4b_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028139fa7f0d5345baaf8850f40f1fcd4b_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,607 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202861b107c043a4c578f19bd7bf65912af_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202861b107c043a4c578f19bd7bf65912af_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,607 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b1ff3c308ed4473a8beb9c9a1c8aeffc_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b1ff3c308ed4473a8beb9c9a1c8aeffc_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,608 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bea67f6b416f4df4a38bcb6e7a04506d_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bea67f6b416f4df4a38bcb6e7a04506d_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,608 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d237bc9b8e884a359fbe985a16a1ae76_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d237bc9b8e884a359fbe985a16a1ae76_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,610 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ddb37092439a4ac6b83c5af7037b533a_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ddb37092439a4ac6b83c5af7037b533a_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,611 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e441ec9761cd442db52fb92c1cd6d2da_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e441ec9761cd442db52fb92c1cd6d2da_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,612 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f5b4a819ea464609ac8ea682b5620c58_1160304dbb0b6d1b48d2c66c029eb555 to hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f5b4a819ea464609ac8ea682b5620c58_1160304dbb0b6d1b48d2c66c029eb555 2024-12-02T05:19:46,612 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T05:19:46,629 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=178, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:46,633 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T05:19:46,636 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T05:19:46,638 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=178, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:46,638 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T05:19:46,638 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733116786638"}]},"ts":"9223372036854775807"} 2024-12-02T05:19:46,641 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T05:19:46,641 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1160304dbb0b6d1b48d2c66c029eb555, NAME => 'TestAcidGuarantees,,1733116757933.1160304dbb0b6d1b48d2c66c029eb555.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T05:19:46,641 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T05:19:46,642 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733116786641"}]},"ts":"9223372036854775807"} 2024-12-02T05:19:46,643 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T05:19:46,644 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=178, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T05:19:46,645 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 94 msec 2024-12-02T05:19:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-02T05:19:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T05:19:46,668 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T05:19:46,682 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252 (was 250) - Thread LEAK? -, OpenFileDescriptor=457 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=572 (was 582), ProcessCount=11 (was 11), AvailableMemoryMB=6451 (was 5814) - AvailableMemoryMB LEAK? - 2024-12-02T05:19:46,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T05:19:46,682 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T05:19:46,683 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T05:19:46,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:46,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:46,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T05:19:46,683 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T05:19:46,683 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1133720664, stopped=false 2024-12-02T05:19:46,683 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f49461030c22,42731,1733116593060 2024-12-02T05:19:46,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T05:19:46,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T05:19:46,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T05:19:46,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T05:19:46,854 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T05:19:46,854 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T05:19:46,854 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T05:19:46,855 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T05:19:46,855 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T05:19:46,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:46,856 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f49461030c22,46495,1733116594459' ***** 2024-12-02T05:19:46,856 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T05:19:46,856 INFO [RS:0;f49461030c22:46495 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T05:19:46,856 INFO [RS:0;f49461030c22:46495 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T05:19:46,856 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T05:19:46,856 INFO [RS:0;f49461030c22:46495 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T05:19:46,856 INFO [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(959): stopping server f49461030c22,46495,1733116594459 2024-12-02T05:19:46,856 INFO [RS:0;f49461030c22:46495 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T05:19:46,856 INFO [RS:0;f49461030c22:46495 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f49461030c22:46495. 2024-12-02T05:19:46,857 DEBUG [RS:0;f49461030c22:46495 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T05:19:46,857 DEBUG [RS:0;f49461030c22:46495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:46,857 INFO [RS:0;f49461030c22:46495 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T05:19:46,857 INFO [RS:0;f49461030c22:46495 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T05:19:46,857 INFO [RS:0;f49461030c22:46495 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T05:19:46,857 INFO [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T05:19:46,858 INFO [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-02T05:19:46,858 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-02T05:19:46,858 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T05:19:46,858 INFO [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T05:19:46,858 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T05:19:46,858 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T05:19:46,858 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T05:19:46,858 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T05:19:46,858 INFO [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-02T05:19:46,901 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/.tmp/info/d806997695214d5d86734ff2c0ca81f7 is 95, key is TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288./info:/1733116631300/DeleteFamily/seqid=0 2024-12-02T05:19:46,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742444_1620 (size=6514) 2024-12-02T05:19:46,930 INFO [regionserver/f49461030c22:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T05:19:47,058 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T05:19:47,258 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T05:19:47,305 INFO [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/.tmp/info/d806997695214d5d86734ff2c0ca81f7 2024-12-02T05:19:47,330 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/.tmp/ns/c71de3f218c34edb8ea2810f4cb248bf is 93, key is TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288./ns:/1733116631264/DeleteFamily/seqid=0 2024-12-02T05:19:47,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742445_1621 (size=5906) 2024-12-02T05:19:47,459 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T05:19:47,659 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T05:19:47,734 INFO [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/.tmp/ns/c71de3f218c34edb8ea2810f4cb248bf 2024-12-02T05:19:47,748 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/.tmp/rep_barrier/7df4b4e4a03c4fe6b6a67cf190dc8808 is 102, key is TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288./rep_barrier:/1733116631264/DeleteFamily/seqid=0 2024-12-02T05:19:47,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742446_1622 (size=6025) 2024-12-02T05:19:47,842 INFO [regionserver/f49461030c22:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T05:19:47,842 INFO [regionserver/f49461030c22:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T05:19:47,859 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T05:19:48,059 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T05:19:48,152 INFO [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/.tmp/rep_barrier/7df4b4e4a03c4fe6b6a67cf190dc8808 2024-12-02T05:19:48,174 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/.tmp/table/ee7f43f40565452da18a6b9abb6bad54 is 96, key is TestAcidGuarantees,,1733116601545.d2d081cdce71d4cc4bebd39e9655d288./table:/1733116631264/DeleteFamily/seqid=0 2024-12-02T05:19:48,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742447_1623 (size=5936) 2024-12-02T05:19:48,259 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T05:19:48,460 DEBUG [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T05:19:48,578 INFO [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/.tmp/table/ee7f43f40565452da18a6b9abb6bad54 2024-12-02T05:19:48,584 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/.tmp/info/d806997695214d5d86734ff2c0ca81f7 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/info/d806997695214d5d86734ff2c0ca81f7 2024-12-02T05:19:48,588 INFO [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/info/d806997695214d5d86734ff2c0ca81f7, entries=12, sequenceid=90, filesize=6.4 K 2024-12-02T05:19:48,589 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/.tmp/ns/c71de3f218c34edb8ea2810f4cb248bf as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/ns/c71de3f218c34edb8ea2810f4cb248bf 2024-12-02T05:19:48,592 INFO [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/ns/c71de3f218c34edb8ea2810f4cb248bf, entries=8, sequenceid=90, filesize=5.8 K 2024-12-02T05:19:48,593 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/.tmp/rep_barrier/7df4b4e4a03c4fe6b6a67cf190dc8808 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/rep_barrier/7df4b4e4a03c4fe6b6a67cf190dc8808 2024-12-02T05:19:48,596 INFO [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/rep_barrier/7df4b4e4a03c4fe6b6a67cf190dc8808, entries=6, sequenceid=90, filesize=5.9 K 2024-12-02T05:19:48,597 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/.tmp/table/ee7f43f40565452da18a6b9abb6bad54 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/table/ee7f43f40565452da18a6b9abb6bad54 2024-12-02T05:19:48,600 INFO [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/table/ee7f43f40565452da18a6b9abb6bad54, entries=7, sequenceid=90, filesize=5.8 K 2024-12-02T05:19:48,601 INFO [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1743ms, sequenceid=90, compaction requested=false 2024-12-02T05:19:48,606 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-02T05:19:48,607 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T05:19:48,607 INFO [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T05:19:48,607 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733116786858Running coprocessor pre-close hooks at 1733116786858Disabling compacts and flushes for region at 1733116786858Disabling writes for close at 1733116786858Obtaining lock to block concurrent updates at 1733116786858Preparing flush snapshotting stores in 1588230740 at 1733116786858Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733116786859 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733116786877 (+18 ms)Flushing 1588230740/info: creating writer at 1733116786877Flushing 1588230740/info: appending metadata at 1733116786901 (+24 ms)Flushing 1588230740/info: closing flushed file at 1733116786901Flushing 1588230740/ns: creating writer at 1733116787313 (+412 ms)Flushing 1588230740/ns: appending metadata at 1733116787330 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733116787330Flushing 1588230740/rep_barrier: creating writer at 1733116787736 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733116787748 (+12 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733116787748Flushing 1588230740/table: creating writer at 1733116788156 (+408 ms)Flushing 1588230740/table: appending metadata at 1733116788174 (+18 ms)Flushing 1588230740/table: closing flushed file at 1733116788174Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4aa977a3: reopening flushed file at 1733116788583 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@510a3fc7: reopening flushed file at 1733116788589 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fffd9dd: reopening flushed file at 1733116788593 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9b0f1ac: reopening flushed file at 1733116788596 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1743ms, sequenceid=90, compaction requested=false at 1733116788601 (+5 ms)Writing region close event to WAL at 1733116788603 (+2 ms)Running coprocessor post-close hooks at 1733116788607 (+4 ms)Closed at 1733116788607 2024-12-02T05:19:48,608 DEBUG [RS_CLOSE_META-regionserver/f49461030c22:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T05:19:48,660 INFO [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(976): stopping server f49461030c22,46495,1733116594459; all regions closed. 2024-12-02T05:19:48,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741834_1010 (size=25344) 2024-12-02T05:19:48,671 DEBUG [RS:0;f49461030c22:46495 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/oldWALs 2024-12-02T05:19:48,671 INFO [RS:0;f49461030c22:46495 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f49461030c22%2C46495%2C1733116594459.meta:.meta(num 1733116600988) 2024-12-02T05:19:48,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741833_1009 (size=15524723) 2024-12-02T05:19:48,676 DEBUG [RS:0;f49461030c22:46495 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/oldWALs 2024-12-02T05:19:48,676 INFO [RS:0;f49461030c22:46495 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f49461030c22%2C46495%2C1733116594459:(num 1733116599990) 2024-12-02T05:19:48,676 DEBUG [RS:0;f49461030c22:46495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T05:19:48,676 INFO [RS:0;f49461030c22:46495 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T05:19:48,676 INFO [RS:0;f49461030c22:46495 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T05:19:48,677 INFO [RS:0;f49461030c22:46495 {}] hbase.ChoreService(370): Chore service for: regionserver/f49461030c22:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T05:19:48,677 INFO [RS:0;f49461030c22:46495 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T05:19:48,677 INFO [regionserver/f49461030c22:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T05:19:48,678 INFO [RS:0;f49461030c22:46495 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46495 2024-12-02T05:19:48,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T05:19:48,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f49461030c22,46495,1733116594459 2024-12-02T05:19:48,738 INFO [RS:0;f49461030c22:46495 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T05:19:48,739 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f49461030c22,46495,1733116594459] 2024-12-02T05:19:48,816 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f49461030c22,46495,1733116594459 already deleted, retry=false 2024-12-02T05:19:48,817 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f49461030c22,46495,1733116594459 expired; onlineServers=0 2024-12-02T05:19:48,818 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f49461030c22,42731,1733116593060' ***** 2024-12-02T05:19:48,818 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T05:19:48,818 INFO [M:0;f49461030c22:42731 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T05:19:48,818 INFO [M:0;f49461030c22:42731 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T05:19:48,819 DEBUG [M:0;f49461030c22:42731 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T05:19:48,819 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T05:19:48,819 DEBUG [M:0;f49461030c22:42731 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T05:19:48,819 DEBUG [master/f49461030c22:0:becomeActiveMaster-HFileCleaner.large.0-1733116599481 {}] cleaner.HFileCleaner(306): Exit Thread[master/f49461030c22:0:becomeActiveMaster-HFileCleaner.large.0-1733116599481,5,FailOnTimeoutGroup] 2024-12-02T05:19:48,819 DEBUG [master/f49461030c22:0:becomeActiveMaster-HFileCleaner.small.0-1733116599482 {}] cleaner.HFileCleaner(306): Exit Thread[master/f49461030c22:0:becomeActiveMaster-HFileCleaner.small.0-1733116599482,5,FailOnTimeoutGroup] 2024-12-02T05:19:48,820 INFO [M:0;f49461030c22:42731 {}] hbase.ChoreService(370): Chore service for: master/f49461030c22:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T05:19:48,820 INFO [M:0;f49461030c22:42731 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T05:19:48,820 DEBUG [M:0;f49461030c22:42731 {}] master.HMaster(1795): Stopping service threads 2024-12-02T05:19:48,821 INFO [M:0;f49461030c22:42731 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T05:19:48,821 INFO [M:0;f49461030c22:42731 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T05:19:48,822 ERROR [M:0;f49461030c22:42731 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (2068722025) connection to localhost/127.0.0.1:41031 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:41031,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-02T05:19:48,823 INFO [M:0;f49461030c22:42731 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T05:19:48,823 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T05:19:48,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T05:19:48,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46495-0x10194787add0001, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T05:19:48,886 INFO [RS:0;f49461030c22:46495 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T05:19:48,886 INFO [RS:0;f49461030c22:46495 {}] regionserver.HRegionServer(1031): Exiting; stopping=f49461030c22,46495,1733116594459; zookeeper connection closed. 2024-12-02T05:19:48,887 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4236e16 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4236e16 2024-12-02T05:19:48,888 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-02T05:19:48,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T05:19:48,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T05:19:48,890 DEBUG [M:0;f49461030c22:42731 {}] zookeeper.ZKUtil(347): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T05:19:48,891 WARN [M:0;f49461030c22:42731 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T05:19:48,891 INFO [M:0;f49461030c22:42731 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/.lastflushedseqids 2024-12-02T05:19:48,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742448_1624 (size=72) 2024-12-02T05:19:49,300 INFO [M:0;f49461030c22:42731 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T05:19:49,300 INFO [M:0;f49461030c22:42731 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T05:19:49,300 DEBUG [M:0;f49461030c22:42731 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T05:19:49,300 INFO [M:0;f49461030c22:42731 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T05:19:49,300 DEBUG [M:0;f49461030c22:42731 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T05:19:49,300 DEBUG [M:0;f49461030c22:42731 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T05:19:49,300 DEBUG [M:0;f49461030c22:42731 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T05:19:49,300 INFO [M:0;f49461030c22:42731 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=775.74 KB heapSize=950.12 KB 2024-12-02T05:19:49,314 DEBUG [M:0;f49461030c22:42731 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eb3781256bd341c8944eac3acfba11e9 is 82, key is hbase:meta,,1/info:regioninfo/1733116601069/Put/seqid=0 2024-12-02T05:19:49,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742449_1625 (size=5672) 2024-12-02T05:19:49,718 INFO [M:0;f49461030c22:42731 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2145 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eb3781256bd341c8944eac3acfba11e9 2024-12-02T05:19:49,742 DEBUG [M:0;f49461030c22:42731 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/330f0a7c686346bebcbbe4e9bb3b79f8 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1733116761023/Put/seqid=0 2024-12-02T05:19:49,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742450_1626 (size=46818) 2024-12-02T05:19:50,146 INFO [M:0;f49461030c22:42731 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=775.18 KB at sequenceid=2145 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/330f0a7c686346bebcbbe4e9bb3b79f8 2024-12-02T05:19:50,154 INFO [M:0;f49461030c22:42731 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 330f0a7c686346bebcbbe4e9bb3b79f8 2024-12-02T05:19:50,168 DEBUG [M:0;f49461030c22:42731 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e9216ce1f5d34014a7b48bdfcaf4a314 is 69, key is f49461030c22,46495,1733116594459/rs:state/1733116599561/Put/seqid=0 2024-12-02T05:19:50,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073742451_1627 (size=5156) 2024-12-02T05:19:50,412 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T05:19:50,572 INFO [M:0;f49461030c22:42731 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2145 (bloomFilter=true), to=hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e9216ce1f5d34014a7b48bdfcaf4a314 2024-12-02T05:19:50,579 DEBUG [M:0;f49461030c22:42731 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eb3781256bd341c8944eac3acfba11e9 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/eb3781256bd341c8944eac3acfba11e9 2024-12-02T05:19:50,585 INFO [M:0;f49461030c22:42731 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/eb3781256bd341c8944eac3acfba11e9, entries=8, sequenceid=2145, filesize=5.5 K 2024-12-02T05:19:50,586 DEBUG [M:0;f49461030c22:42731 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/330f0a7c686346bebcbbe4e9bb3b79f8 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/330f0a7c686346bebcbbe4e9bb3b79f8 2024-12-02T05:19:50,591 INFO [M:0;f49461030c22:42731 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 330f0a7c686346bebcbbe4e9bb3b79f8 2024-12-02T05:19:50,591 INFO [M:0;f49461030c22:42731 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/330f0a7c686346bebcbbe4e9bb3b79f8, entries=178, sequenceid=2145, filesize=45.7 K 2024-12-02T05:19:50,592 DEBUG [M:0;f49461030c22:42731 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e9216ce1f5d34014a7b48bdfcaf4a314 as hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e9216ce1f5d34014a7b48bdfcaf4a314 2024-12-02T05:19:50,595 INFO [M:0;f49461030c22:42731 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41031/user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e9216ce1f5d34014a7b48bdfcaf4a314, entries=1, sequenceid=2145, filesize=5.0 K 2024-12-02T05:19:50,595 INFO [M:0;f49461030c22:42731 {}] regionserver.HRegion(3140): Finished flush of dataSize ~775.74 KB/794355, heapSize ~949.82 KB/972616, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1295ms, sequenceid=2145, compaction requested=false 2024-12-02T05:19:50,597 INFO [M:0;f49461030c22:42731 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T05:19:50,597 DEBUG [M:0;f49461030c22:42731 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733116789300Disabling compacts and flushes for region at 1733116789300Disabling writes for close at 1733116789300Obtaining lock to block concurrent updates at 1733116789300Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733116789300Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=794355, getHeapSize=972856, getOffHeapSize=0, getCellsCount=2348 at 1733116789300Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733116789301 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733116789301Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733116789313 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733116789313Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733116789725 (+412 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733116789741 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733116789741Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733116790154 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733116790168 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733116790168Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e2f8e64: reopening flushed file at 1733116790578 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e10bb59: reopening flushed file at 1733116790586 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d983872: reopening flushed file at 1733116790591 (+5 ms)Finished flush of dataSize ~775.74 KB/794355, heapSize ~949.82 KB/972616, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1295ms, sequenceid=2145, compaction requested=false at 1733116790595 (+4 ms)Writing region close event to WAL at 1733116790597 (+2 ms)Closed at 1733116790597 2024-12-02T05:19:50,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34519 is added to blk_1073741830_1006 (size=935011) 2024-12-02T05:19:50,599 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/b157354e-d4b2-0fb3-c968-1dfaa2df596c/MasterData/WALs/f49461030c22,42731,1733116593060/f49461030c22%2C42731%2C1733116593060.1733116597483 not finished, retry = 0 2024-12-02T05:19:50,700 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T05:19:50,701 INFO [M:0;f49461030c22:42731 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T05:19:50,701 INFO [M:0;f49461030c22:42731 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42731 2024-12-02T05:19:50,701 INFO [M:0;f49461030c22:42731 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T05:19:50,816 INFO [M:0;f49461030c22:42731 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T05:19:50,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T05:19:50,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42731-0x10194787add0000, quorum=127.0.0.1:50310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T05:19:50,823 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7acb84b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T05:19:50,826 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T05:19:50,827 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T05:19:50,827 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T05:19:50,827 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/hadoop.log.dir/,STOPPED} 2024-12-02T05:19:50,831 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T05:19:50,831 WARN [BP-16702269-172.17.0.2-1733116583194 heartbeating to localhost/127.0.0.1:41031 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T05:19:50,831 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T05:19:50,831 WARN [BP-16702269-172.17.0.2-1733116583194 heartbeating to localhost/127.0.0.1:41031 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-16702269-172.17.0.2-1733116583194 (Datanode Uuid 3d349b4e-6cdf-4ef7-bca2-2a33ed5b96a5) service to localhost/127.0.0.1:41031 2024-12-02T05:19:50,833 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/cluster_01f97ee8-23a4-f5a0-8cda-7a5425401614/data/data1/current/BP-16702269-172.17.0.2-1733116583194 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T05:19:50,833 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/cluster_01f97ee8-23a4-f5a0-8cda-7a5425401614/data/data2/current/BP-16702269-172.17.0.2-1733116583194 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T05:19:50,833 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T05:19:50,841 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T05:19:50,841 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T05:19:50,841 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T05:19:50,842 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T05:19:50,842 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ace4b2fc-20b1-fe72-c1b9-405d166fadda/hadoop.log.dir/,STOPPED} 2024-12-02T05:19:50,860 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T05:19:50,991 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down